/*************************************************************************/ /*!
@File
@Title          Common MMU Management
@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
@Description    Implements basic low level control of MMU.
@License        Dual MIT/GPLv2

The contents of this file are subject to the MIT license as set out below.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 2 ("GPL") in which case the provisions
of GPL are applicable instead of those above.

If you wish to allow use of your version of this file only under the terms of
GPL, and not to allow others to use your version of this file under the terms
of the MIT license, indicate your decision by deleting the provisions above
and replace them with the notice and other provisions required by GPL as set
out in the file called "GPL-COPYING" included in this distribution. If you do
not delete the provisions above, a recipient may use your version of this file
under the terms of either the MIT license or GPL.

This License is also included in this distribution in the file called
"MIT-COPYING".

EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */ /***************************************************************************/

#include "devicemem_server_utils.h"

/* Our own interface */
#include "mmu_common.h"

#include "rgx_bvnc_defs_km.h"
#include "rgxmmudefs_km.h"
/*
Interfaces to other modules:

Let's keep this graph up-to-date:

   +-----------+
   | devicemem |
   +-----------+
         |
   +============+
   | mmu_common |
   +============+
         |
         +-----------------+
         |                 |
    +---------+      +----------+
    |   pmr   |      |  device  |
    +---------+      +----------+
 */

#include "img_types.h"
#include "img_defs.h"
#include "osfunc.h"
#include "allocmem.h"
#if defined(PDUMP)
#include "pdump_km.h"
#include "pdump_physmem.h"
#endif
#include "pmr.h"
/* include/ */
#include "pvr_debug.h"
#include "pvr_notifier.h"
#include "pvrsrv_error.h"
#include "pvrsrv.h"
#include "htbuffer.h"

#include "rgxdevice.h"

#if defined(SUPPORT_GPUVIRT_VALIDATION)
#include "physmem_lma.h"
#endif

#include "dllist.h"

// #define MMU_OBJECT_REFCOUNT_DEBUGING 1
#if defined (MMU_OBJECT_REFCOUNT_DEBUGING)
#define MMU_OBJ_DBG(x)	PVR_DPF(x)
#else
#define MMU_OBJ_DBG(x)
#endif

/*!
 * Refcounted structure that is shared between the context and
 * the cleanup thread items.
 * It is used to keep track of all cleanup items and whether the creating
 * MMU context has been destroyed and therefore is not allowed to be
 * accessed any more.
 *
 * The cleanup thread is used to defer the freeing of the page tables
 * because we have to make sure that the MMU cache has been invalidated.
 * If we don't take care of this the MMU might partially access cached
 * and uncached tables which might lead to inconsistencies and in the
 * worst case to MMU pending faults on random memory.
 */
typedef struct _MMU_CTX_CLEANUP_DATA_
{
	/*! Refcount to know when this structure can be destroyed */
	ATOMIC_T iRef;
	/*! Protect items in this structure, especially the refcount */
	POS_LOCK hCleanupLock;
	/*! List of all cleanup items currently in flight */
	DLLIST_NODE sMMUCtxCleanupItemsHead;
	/*! Was the MMU context destroyed and should not be accessed any more? */
	IMG_BOOL bMMUContextExists;
} MMU_CTX_CLEANUP_DATA;


/*!
 * Structure holding one or more page tables that need to be
 * freed after the MMU cache has been flushed which is signalled when
 * the stored sync has a value that is <= the required value.
 */
typedef struct _MMU_CLEANUP_ITEM_
{
	/*! Cleanup thread data */
	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
	/*! List to hold all the MMU_MEMORY_MAPPINGs, i.e. page tables */
	DLLIST_NODE sMMUMappingHead;
	/*! Node of the cleanup item list for the context */
	DLLIST_NODE sMMUCtxCleanupItem;
	/* Pointer to the cleanup meta data */
	MMU_CTX_CLEANUP_DATA *psMMUCtxCleanupData;
	/* Sync to query if the MMU cache was flushed */
	PVRSRV_CLIENT_SYNC_PRIM *psSync;
	/*! The update value of the sync to signal that the cache was flushed */
	IMG_UINT16 uiRequiredSyncVal;
	/*! The device node needed to free the page tables */
	PVRSRV_DEVICE_NODE *psDevNode;
} MMU_CLEANUP_ITEM;

/*!
	All physical allocations and frees are relative to this context, so
	we would get all the allocations of PCs, PDs, and PTs from the same
	RA.

	We have one per MMU context in case we have mixed UMA/LMA devices
	within the same system.
 */
typedef struct _MMU_PHYSMEM_CONTEXT_
{
	/*! Parent device node */
	PVRSRV_DEVICE_NODE *psDevNode;

	/*! Refcount so we know when to free up the arena */
	IMG_UINT32 uiNumAllocations;

	/*! Arena from which physical memory is derived */
	RA_ARENA *psPhysMemRA;
	/*! Arena name */
	IMG_CHAR *pszPhysMemRAName;
	/*! Size of arena name string */
	size_t uiPhysMemRANameAllocSize;

	/*! Meta data for deferred cleanup */
	MMU_CTX_CLEANUP_DATA *psCleanupData;
	/*! Temporary list of all deferred MMU_MEMORY_MAPPINGs. */
	DLLIST_NODE sTmpMMUMappingHead;

} MMU_PHYSMEM_CONTEXT;

/*!
	Mapping structure for MMU memory allocation
 */
typedef struct _MMU_MEMORY_MAPPING_
{
	/*! Physmem context to allocate from */
	MMU_PHYSMEM_CONTEXT		*psContext;
	/*! OS/system Handle for this allocation */
	PG_HANDLE				sMemHandle;
	/*! CPU virtual address of this allocation */
	void					*pvCpuVAddr;
	/*! Device physical address of this allocation */
	IMG_DEV_PHYADDR			sDevPAddr;
	/*! Size of this allocation */
	size_t					uiSize;
	/*! Number of current mappings of this allocation */
	IMG_UINT32				uiCpuVAddrRefCount;
	/*! Node for the defer free list */
	DLLIST_NODE				sMMUMappingItem;
} MMU_MEMORY_MAPPING;

/*!
	Memory descriptor for MMU objects. There can be more than one memory
	descriptor per MMU memory allocation.
 */
typedef struct _MMU_MEMORY_DESC_
{
	/* NB: bValid is set if this descriptor describes physical
	   memory.  This allows "empty" descriptors to exist, such that we
	   can allocate them in batches. */
	/*! Does this MMU object have physical backing */
	IMG_BOOL				bValid;
	/*! Device Physical address of physical backing */
	IMG_DEV_PHYADDR			sDevPAddr;
	/*! CPU virtual address of physical backing */
	void					*pvCpuVAddr;
	/*! Mapping data for this MMU object */
	MMU_MEMORY_MAPPING		*psMapping;
	/*! Memdesc offset into the psMapping */
	IMG_UINT32 uiOffset;
	/*! Size of the Memdesc */
	IMG_UINT32 uiSize;
} MMU_MEMORY_DESC;

/*!
	MMU levelx structure. This is generic and is used
	for all levels (PC, PD, PT).
 */
typedef struct _MMU_Levelx_INFO_
{
	/*! The Number of entries in this level */
	IMG_UINT32 ui32NumOfEntries;

	/*! Number of times this level has been reference. Note: For Level1 (PTE)
	    we still take/drop the reference when setting up the page tables rather
	    then at map/unmap time as this simplifies things */
	IMG_UINT32 ui32RefCount;

	/*! MemDesc for this level */
	MMU_MEMORY_DESC sMemDesc;

	/*! Array of infos for the next level. Must be last member in structure */
	struct _MMU_Levelx_INFO_ *apsNextLevel[1];
} MMU_Levelx_INFO;

/*!
	MMU context structure
 */
struct _MMU_CONTEXT_
{
	/*! Parent device node */
	PVRSRV_DEVICE_NODE *psDevNode;

	MMU_DEVICEATTRIBS *psDevAttrs;

	/*! For allocation and deallocation of the physical memory where
	    the pagetables live */
	struct _MMU_PHYSMEM_CONTEXT_ *psPhysMemCtx;

#if defined(PDUMP)
	/*! PDump context ID (required for PDump commands with virtual addresses) */
	IMG_UINT32 uiPDumpContextID;

	/*! The refcount of the PDump context ID */
	IMG_UINT32 ui32PDumpContextIDRefCount;
#endif

	/*! Data that is passed back during device specific callbacks */
	IMG_HANDLE hDevData;

#if defined(SUPPORT_GPUVIRT_VALIDATION)
	IMG_UINT32  ui32OSid;
	IMG_UINT32	ui32OSidReg;
	IMG_BOOL   bOSidAxiProt;
#endif

	/*! Lock to ensure exclusive access when manipulating the MMU context or
	 * reading and using its content
	 */
	POS_LOCK hLock;

	/*! Base level info structure. Must be last member in structure */
	MMU_Levelx_INFO sBaseLevelInfo;
	/* NO OTHER MEMBERS AFTER THIS STRUCTURE ! */
};

static const IMG_DEV_PHYADDR gsBadDevPhyAddr = {MMU_BAD_PHYS_ADDR};

#if defined(DEBUG)
#include "log2.h"
#endif


/*****************************************************************************
 *                          Utility functions                                *
 *****************************************************************************/

/*************************************************************************/ /*!
@Function       _FreeMMUMapping

@Description    Free a given dllist of MMU_MEMORY_MAPPINGs and the page tables
                they represent.

@Input          psDevNode           Device node

@Input          psTmpMMUMappingHead List of MMU_MEMORY_MAPPINGs to free
 */
/*****************************************************************************/
static void
_FreeMMUMapping(PVRSRV_DEVICE_NODE *psDevNode,
                PDLLIST_NODE psTmpMMUMappingHead)
{
	PDLLIST_NODE psNode, psNextNode;

	/* Free the current list unconditionally */
	dllist_foreach_node(psTmpMMUMappingHead,
	                    psNode,
	                    psNextNode)
	{
		MMU_MEMORY_MAPPING *psMapping = IMG_CONTAINER_OF(psNode,
		                                                 MMU_MEMORY_MAPPING,
		                                                 sMMUMappingItem);

		psDevNode->pfnDevPxFree(psDevNode, &psMapping->sMemHandle);
		dllist_remove_node(psNode);
		OSFreeMem(psMapping);
	}
}

/*************************************************************************/ /*!
@Function       _CleanupThread_FreeMMUMapping

@Description    Function to be executed by the cleanup thread to free
                MMU_MEMORY_MAPPINGs after the MMU cache has been invalidated.

                This function will request a MMU cache invalidate once and
                retry to free the MMU_MEMORY_MAPPINGs until the invalidate
                has been executed.

                If the memory context that created this cleanup item has been
                destroyed in the meantime this function will directly free the
                MMU_MEMORY_MAPPINGs without waiting for any MMU cache
                invalidation.

@Input          pvData           Cleanup data in form of a MMU_CLEANUP_ITEM

@Return         PVRSRV_OK if successful otherwise PVRSRV_ERROR_RETRY
 */
/*****************************************************************************/
static PVRSRV_ERROR
_CleanupThread_FreeMMUMapping(void* pvData)
{
	PVRSRV_ERROR eError;
	MMU_CLEANUP_ITEM *psCleanup = (MMU_CLEANUP_ITEM *) pvData;
	MMU_CTX_CLEANUP_DATA *psMMUCtxCleanupData = psCleanup->psMMUCtxCleanupData;
	PVRSRV_DEVICE_NODE *psDevNode = psCleanup->psDevNode;
	IMG_BOOL bFreeNow;
	IMG_UINT32 uiSyncCurrent;
	IMG_UINT32 uiSyncReq;

	OSLockAcquire(psMMUCtxCleanupData->hCleanupLock);

	/* Don't attempt to free anything when the context has been destroyed.
	 * Especially don't access any device specific structures any more!*/
	if (!psMMUCtxCleanupData->bMMUContextExists)
	{
		OSFreeMem(psCleanup);
		eError = PVRSRV_OK;
		goto e0;
	}

	if (psCleanup->psSync == NULL)
	{
		/* Kick to invalidate the MMU caches and get sync info */
		psDevNode->pfnMMUCacheInvalidateKick(psDevNode,
		                                     &psCleanup->uiRequiredSyncVal,
		                                     IMG_TRUE);
		psCleanup->psSync = psDevNode->psMMUCacheSyncPrim;
	}

	uiSyncCurrent = OSReadDeviceMem32(psCleanup->psSync->pui32LinAddr);
	uiSyncReq = psCleanup->uiRequiredSyncVal;

	/* Either the invalidate has been executed ... */
	bFreeNow = (uiSyncCurrent >= uiSyncReq) ? IMG_TRUE :
			/* ... with the counter wrapped around ... */
			(uiSyncReq - uiSyncCurrent) > 0xEFFFFFFFUL ? IMG_TRUE :
					/* ... or are we still waiting for the invalidate? */
					IMG_FALSE;

#if defined(NO_HARDWARE)
	/* In NOHW the syncs will never be updated so just free the tables */
	bFreeNow = IMG_TRUE;
#endif

	if (bFreeNow)
	{
		_FreeMMUMapping(psDevNode, &psCleanup->sMMUMappingHead);

		dllist_remove_node(&psCleanup->sMMUCtxCleanupItem);
		OSFreeMem(psCleanup);

		eError = PVRSRV_OK;
	}
	else
	{
		eError = PVRSRV_ERROR_RETRY;
	}

	e0:

	/* If this cleanup task has been successfully executed we can
	 * decrease the context cleanup data refcount. Successfully
	 * means here that the MMU_MEMORY_MAPPINGs have been freed by
	 * either this cleanup task of when the MMU context has been
	 * destroyed. */
	if (eError == PVRSRV_OK)
	{
		OSLockRelease(psMMUCtxCleanupData->hCleanupLock);

		if (OSAtomicDecrement(&psMMUCtxCleanupData->iRef) == 0)
		{
			OSLockDestroy(psMMUCtxCleanupData->hCleanupLock);
			OSFreeMem(psMMUCtxCleanupData);
		}
	}
	else
	{
		OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
	}


	return eError;
}

/*************************************************************************/ /*!
@Function       _SetupCleanup_FreeMMUMapping

@Description    Setup a cleanup item for the cleanup thread that will
                kick off a MMU invalidate request and free the associated
                MMU_MEMORY_MAPPINGs when the invalidate was successful.

@Input          psDevNode           Device node

@Input          psPhysMemCtx        The current MMU physmem context
 */
/*****************************************************************************/
static void
_SetupCleanup_FreeMMUMapping(PVRSRV_DEVICE_NODE *psDevNode,
                             MMU_PHYSMEM_CONTEXT *psPhysMemCtx)
{

	MMU_CLEANUP_ITEM *psCleanupItem;
	MMU_CTX_CLEANUP_DATA *psCleanupData = psPhysMemCtx->psCleanupData;

	if (dllist_is_empty(&psPhysMemCtx->sTmpMMUMappingHead))
	{
		goto e0;
	}

#if !defined(SUPPORT_MMU_PENDING_FAULT_PROTECTION)
	/* If users deactivated this we immediately free the page tables */
	goto e1;
#endif

	/* Don't defer the freeing if we are currently unloading the driver
	 * or if the sync has been destroyed */
	if (PVRSRVGetPVRSRVData()->bUnload ||
			psDevNode->psMMUCacheSyncPrim == NULL)
	{
		goto e1;
	}

	/* Allocate a cleanup item */
	psCleanupItem = OSAllocMem(sizeof(*psCleanupItem));
	if (!psCleanupItem)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Failed to get memory for deferred page table cleanup. "
				"Freeing tables immediately",
				__func__));
		goto e1;
	}

	/* Set sync to NULL to indicate we did not interact with
	 * the FW yet. Kicking off an MMU cache invalidate should
	 * be done in the cleanup thread to not waste time here. */
	psCleanupItem->psSync = NULL;
	psCleanupItem->uiRequiredSyncVal = 0;
	psCleanupItem->psDevNode = psDevNode;
	psCleanupItem->psMMUCtxCleanupData = psCleanupData;

	OSAtomicIncrement(&psCleanupData->iRef);

	/* Move the page tables to free to the cleanup item */
	dllist_replace_head(&psPhysMemCtx->sTmpMMUMappingHead,
	                    &psCleanupItem->sMMUMappingHead);

	/* Add the cleanup item itself to the context list */
	dllist_add_to_tail(&psCleanupData->sMMUCtxCleanupItemsHead,
	                   &psCleanupItem->sMMUCtxCleanupItem);

	/* Setup the cleanup thread data and add the work item */
	psCleanupItem->sCleanupThreadFn.pfnFree = _CleanupThread_FreeMMUMapping;
	psCleanupItem->sCleanupThreadFn.pvData = psCleanupItem;
	psCleanupItem->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
	CLEANUP_THREAD_SET_RETRY_TIMEOUT(&psCleanupItem->sCleanupThreadFn,
	                                 CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT);

	PVRSRVCleanupThreadAddWork(&psCleanupItem->sCleanupThreadFn);

	return;

	e1:
	/* Free the page tables now */
	_FreeMMUMapping(psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
	e0:
	return;
}

/*************************************************************************/ /*!
@Function       _CalcPCEIdx

@Description    Calculate the page catalogue index

@Input          sDevVAddr           Device virtual address

@Input          psDevVAddrConfig    Configuration of the virtual address

@Input          bRoundUp            Round up the index

@Return         The page catalogue index
 */
/*****************************************************************************/
static IMG_UINT32 _CalcPCEIdx(IMG_DEV_VIRTADDR sDevVAddr,
                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
                              IMG_BOOL bRoundUp)
{
	IMG_DEV_VIRTADDR sTmpDevVAddr;
	IMG_UINT32 ui32RetVal;

	sTmpDevVAddr = sDevVAddr;

	if (bRoundUp)
	{
		sTmpDevVAddr.uiAddr --;
	}
	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPCIndexMask)
			>> psDevVAddrConfig->uiPCIndexShift);

	if (bRoundUp)
	{
		ui32RetVal ++;
	}

	return ui32RetVal;
}


/*************************************************************************/ /*!
@Function       _CalcPDEIdx

@Description    Calculate the page directory index

@Input          sDevVAddr           Device virtual address

@Input          psDevVAddrConfig    Configuration of the virtual address

@Input          bRoundUp            Round up the index

@Return         The page directory index
 */
/*****************************************************************************/
static IMG_UINT32 _CalcPDEIdx(IMG_DEV_VIRTADDR sDevVAddr,
                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
                              IMG_BOOL bRoundUp)
{
	IMG_DEV_VIRTADDR sTmpDevVAddr;
	IMG_UINT32 ui32RetVal;

	sTmpDevVAddr = sDevVAddr;

	if (bRoundUp)
	{
		sTmpDevVAddr.uiAddr --;
	}
	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPDIndexMask)
			>> psDevVAddrConfig->uiPDIndexShift);

	if (bRoundUp)
	{
		ui32RetVal ++;
	}

	return ui32RetVal;
}


/*************************************************************************/ /*!
@Function       _CalcPTEIdx

@Description    Calculate the page entry index

@Input          sDevVAddr           Device virtual address

@Input          psDevVAddrConfig    Configuration of the virtual address

@Input          bRoundUp            Round up the index

@Return         The page entry index
 */
/*****************************************************************************/
static IMG_UINT32 _CalcPTEIdx(IMG_DEV_VIRTADDR sDevVAddr,
                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
                              IMG_BOOL bRoundUp)
{
	IMG_DEV_VIRTADDR sTmpDevVAddr;
	IMG_UINT32 ui32RetVal;

	sTmpDevVAddr = sDevVAddr;
	sTmpDevVAddr.uiAddr -= psDevVAddrConfig->uiOffsetInBytes;
	if (bRoundUp)
	{
		sTmpDevVAddr.uiAddr --;
	}
	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPTIndexMask)
			>> psDevVAddrConfig->uiPTIndexShift);

	if (bRoundUp)
	{
		ui32RetVal ++;
	}

	return ui32RetVal;
}

/*****************************************************************************
 *         MMU memory allocation/management functions (mem desc)             *
 *****************************************************************************/

/*************************************************************************/ /*!
@Function       _MMU_PhysMem_RAImportAlloc

@Description    Imports MMU Px memory into the RA. This is where the
                actual allocation of physical memory happens.

@Input          hArenaHandle    Handle that was passed in during the
                                creation of the RA

@Input          uiSize          Size of the memory to import

@Input          uiFlags         Flags that where passed in the allocation.

@Output         puiBase         The address of where to insert this import

@Output         puiActualSize   The actual size of the import

@Output         phPriv          Handle which will be passed back when
                                this import is freed

@Return         PVRSRV_OK if import alloc was successful
 */
/*****************************************************************************/
static PVRSRV_ERROR _MMU_PhysMem_RAImportAlloc(RA_PERARENA_HANDLE hArenaHandle,
                                               RA_LENGTH_T uiSize,
                                               RA_FLAGS_T uiFlags,
                                               const IMG_CHAR *pszAnnotation,
                                               RA_BASE_T *puiBase,
                                               RA_LENGTH_T *puiActualSize,
                                               RA_PERISPAN_HANDLE *phPriv)
{
	MMU_PHYSMEM_CONTEXT *psCtx = (MMU_PHYSMEM_CONTEXT *) hArenaHandle;
	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *) psCtx->psDevNode;
	MMU_MEMORY_MAPPING *psMapping;
	PVRSRV_ERROR eError;

	PVR_UNREFERENCED_PARAMETER(pszAnnotation);
	PVR_UNREFERENCED_PARAMETER(uiFlags);

	psMapping = OSAllocMem(sizeof(MMU_MEMORY_MAPPING));
	if (psMapping == NULL)
	{
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e0;
	}

	eError = psDevNode->pfnDevPxAlloc(psDevNode, TRUNCATE_64BITS_TO_SIZE_T(uiSize), &psMapping->sMemHandle,
	                                  &psMapping->sDevPAddr);
	if (eError != PVRSRV_OK)
	{
		goto e1;
	}

	psMapping->psContext = psCtx;
	psMapping->uiSize = TRUNCATE_64BITS_TO_SIZE_T(uiSize);

	psMapping->uiCpuVAddrRefCount = 0;

	*phPriv = (RA_PERISPAN_HANDLE) psMapping;

	/* Note: This assumes this memory never gets paged out */
	*puiBase = (RA_BASE_T)psMapping->sDevPAddr.uiAddr;
	*puiActualSize = uiSize;

	return PVRSRV_OK;

	e1:
	OSFreeMem(psMapping);
	e0:
	return eError;
}

/*************************************************************************/ /*!
@Function       _MMU_PhysMem_RAImportFree

@Description    Imports MMU Px memory into the RA. This is where the
                actual free of physical memory happens.

@Input          hArenaHandle    Handle that was passed in during the
                                creation of the RA

@Input          puiBase         The address of where to insert this import

@Output         phPriv          Private data that the import alloc provided

@Return         None
 */
/*****************************************************************************/
static void _MMU_PhysMem_RAImportFree(RA_PERARENA_HANDLE hArenaHandle,
                                      RA_BASE_T uiBase,
                                      RA_PERISPAN_HANDLE hPriv)
{
	MMU_MEMORY_MAPPING *psMapping = (MMU_MEMORY_MAPPING *) hPriv;
	MMU_PHYSMEM_CONTEXT *psCtx = (MMU_PHYSMEM_CONTEXT *) hArenaHandle;

	PVR_UNREFERENCED_PARAMETER(uiBase);

	/* Check we have dropped all CPU mappings */
	PVR_ASSERT(psMapping->uiCpuVAddrRefCount == 0);

	/* Add mapping to defer free list */
	psMapping->psContext = NULL;
	dllist_add_to_tail(&psCtx->sTmpMMUMappingHead, &psMapping->sMMUMappingItem);
}

/*************************************************************************/ /*!
@Function       _MMU_PhysMemAlloc

@Description    Allocates physical memory for MMU objects

@Input          psCtx           Physmem context to do the allocation from

@Output         psMemDesc       Allocation description

@Input          uiBytes         Size of the allocation in bytes

@Input          uiAlignment     Alignment requirement of this allocation

@Return         PVRSRV_OK if allocation was successful
 */
/*****************************************************************************/

static PVRSRV_ERROR _MMU_PhysMemAlloc(MMU_PHYSMEM_CONTEXT *psCtx,
                                      MMU_MEMORY_DESC *psMemDesc,
                                      size_t uiBytes,
                                      size_t uiAlignment)
{
	PVRSRV_ERROR eError;
	RA_BASE_T uiPhysAddr;

	if (!psMemDesc || psMemDesc->bValid)
	{
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	eError = RA_Alloc(psCtx->psPhysMemRA,
	                  uiBytes,
	                  RA_NO_IMPORT_MULTIPLIER,
	                  0, /* flags */
	                  uiAlignment,
	                  "",
	                  &uiPhysAddr,
	                  NULL,
	                  (RA_PERISPAN_HANDLE *) &psMemDesc->psMapping);

	PVR_LOGR_IF_ERROR(eError, "RA_Alloc");

	psMemDesc->bValid = IMG_TRUE;
	psMemDesc->pvCpuVAddr = NULL;
	psMemDesc->sDevPAddr.uiAddr = (IMG_UINT64) uiPhysAddr;

	if (psMemDesc->psMapping->uiCpuVAddrRefCount == 0)
	{
		eError = psCtx->psDevNode->pfnDevPxMap(psCtx->psDevNode,
		                                       &psMemDesc->psMapping->sMemHandle,
		                                       psMemDesc->psMapping->uiSize,
		                                       &psMemDesc->psMapping->sDevPAddr,
		                                       &psMemDesc->psMapping->pvCpuVAddr);
		if (eError != PVRSRV_OK)
		{
			RA_Free(psCtx->psPhysMemRA, psMemDesc->sDevPAddr.uiAddr);
			return eError;
		}
	}

	psMemDesc->psMapping->uiCpuVAddrRefCount++;
	psMemDesc->uiOffset = (psMemDesc->sDevPAddr.uiAddr - psMemDesc->psMapping->sDevPAddr.uiAddr);
	psMemDesc->pvCpuVAddr = (IMG_UINT8 *) psMemDesc->psMapping->pvCpuVAddr + psMemDesc->uiOffset;
	psMemDesc->uiSize = uiBytes;
	PVR_ASSERT(psMemDesc->pvCpuVAddr != NULL);

	return PVRSRV_OK;
}

/*************************************************************************/ /*!
@Function       _MMU_PhysMemFree

@Description    Allocates physical memory for MMU objects

@Input          psCtx           Physmem context to do the free on

@Input          psMemDesc       Allocation description

@Return         None
 */
/*****************************************************************************/
static void _MMU_PhysMemFree(MMU_PHYSMEM_CONTEXT *psCtx,
                             MMU_MEMORY_DESC *psMemDesc)
{
	RA_BASE_T uiPhysAddr;

	PVR_ASSERT(psMemDesc->bValid);

	if (--psMemDesc->psMapping->uiCpuVAddrRefCount == 0)
	{
		psCtx->psDevNode->pfnDevPxUnMap(psCtx->psDevNode, &psMemDesc->psMapping->sMemHandle,
		                                psMemDesc->psMapping->pvCpuVAddr);
	}

	psMemDesc->pvCpuVAddr = NULL;

	uiPhysAddr = psMemDesc->sDevPAddr.uiAddr;
	RA_Free(psCtx->psPhysMemRA, uiPhysAddr);

	psMemDesc->bValid = IMG_FALSE;
}


/*****************************************************************************
 *              MMU object allocation/management functions                   *
 *****************************************************************************/

static INLINE PVRSRV_ERROR _MMU_ConvertDevMemFlags(IMG_BOOL bInvalidate,
                                                   PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
                                                   MMU_PROTFLAGS_T *uiMMUProtFlags,
                                                   MMU_CONTEXT *psMMUContext)
{
	PVRSRV_ERROR eError = PVRSRV_OK;
	IMG_UINT32 uiGPUCacheMode;

	/* Do flag conversion between devmem flags and MMU generic flags */
	if (bInvalidate == IMG_FALSE)
	{
		*uiMMUProtFlags |= ((uiMappingFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK)
				>> PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET)
				<< MMU_PROTFLAGS_DEVICE_OFFSET;

		if (PVRSRV_CHECK_GPU_READABLE(uiMappingFlags))
		{
			*uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
		}
		if (PVRSRV_CHECK_GPU_WRITEABLE(uiMappingFlags))
		{
			*uiMMUProtFlags |= MMU_PROTFLAGS_WRITEABLE;
		}

		eError = DevmemDeviceCacheMode(psMMUContext->psDevNode,
		                               uiMappingFlags,
		                               &uiGPUCacheMode);
		if (eError != PVRSRV_OK)
		{
			return eError;
		}

		switch (uiGPUCacheMode)
		{
			case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED:
			case PVRSRV_MEMALLOCFLAG_GPU_WRITE_COMBINE:
				break;
			case PVRSRV_MEMALLOCFLAG_GPU_CACHED:
				*uiMMUProtFlags |= MMU_PROTFLAGS_CACHED;
				break;
			default:
				PVR_DPF((PVR_DBG_ERROR,
						"%s: Wrong parameters",
						__func__));
				return PVRSRV_ERROR_INVALID_PARAMS;
		}

		if (DevmemDeviceCacheCoherency(psMMUContext->psDevNode, uiMappingFlags))
		{
			*uiMMUProtFlags |= MMU_PROTFLAGS_CACHE_COHERENT;
		}

#if defined(SUPPORT_RGX)
		if ((psMMUContext->psDevNode->pfnCheckDeviceFeature) &&
			 PVRSRV_IS_FEATURE_SUPPORTED(psMMUContext->psDevNode, MIPS))
		{
			/*
				If we are allocating on the MMU of the firmware processor, the cached/uncached attributes
				must depend on the FIRMWARE_CACHED allocation flag.
			 */
			if (psMMUContext->psDevAttrs == psMMUContext->psDevNode->psFirmwareMMUDevAttrs)
			{
				if (uiMappingFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED))
				{
					*uiMMUProtFlags |= MMU_PROTFLAGS_CACHED;
				}
				else
				{
					*uiMMUProtFlags &= ~MMU_PROTFLAGS_CACHED;

				}
				*uiMMUProtFlags &= ~MMU_PROTFLAGS_CACHE_COHERENT;
			}
		}
#endif
	}
	else
	{
		*uiMMUProtFlags |= MMU_PROTFLAGS_INVALID;
	}

	return PVRSRV_OK;
}

/*************************************************************************/ /*!
@Function       _PxMemAlloc

@Description    Allocates physical memory for MMU objects, initialises
                and PDumps it.

@Input          psMMUContext    MMU context

@Input          uiNumEntries    Number of entries to allocate

@Input          psConfig        MMU Px config

@Input          eMMULevel       MMU level that that allocation is for

@Output         psMemDesc       Description of allocation

@Return         PVRSRV_OK if allocation was successful
 */
/*****************************************************************************/
static PVRSRV_ERROR _PxMemAlloc(MMU_CONTEXT *psMMUContext,
                                IMG_UINT32 uiNumEntries,
                                const MMU_PxE_CONFIG *psConfig,
                                MMU_LEVEL eMMULevel,
                                MMU_MEMORY_DESC *psMemDesc,
                                IMG_UINT32 uiLog2Align)
{
	PVRSRV_ERROR eError;
	size_t uiBytes;
	size_t uiAlign;

	PVR_ASSERT(psConfig->uiBytesPerEntry != 0);

	uiBytes = uiNumEntries * psConfig->uiBytesPerEntry;
	/* We need here the alignment of the previous level because that is the entry for we generate here */
	uiAlign = 1 << uiLog2Align;

	/*
	 * If the hardware specifies an alignment requirement for a page table then
	 * it also requires that all memory up to the next aligned address is
	 * zeroed.
	 *
	 * Failing to do this can result in uninitialised data outside of the actual
	 * page table range being read by the MMU and treated as valid, e.g. the
	 * pending flag.
	 *
	 * Typically this will affect 1MiB, 2MiB PT pages which have a size of 16
	 * and 8 bytes respectively but an alignment requirement of 64 bytes each.
	 */
	uiBytes = PVR_ALIGN(uiBytes, uiAlign);

	/* allocate the object */
	eError = _MMU_PhysMemAlloc(psMMUContext->psPhysMemCtx,
	                           psMemDesc, uiBytes, uiAlign);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "_PxMemAlloc: failed to allocate memory for the MMU object"));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e0;
	}

	/*
		Clear the object
		Note: if any MMUs are cleared with non-zero values then will need a
		custom clear function
		Note: 'Cached' is wrong for the LMA + ARM64 combination, but this is
		unlikely
	 */
	OSCachedMemSet(psMemDesc->pvCpuVAddr, 0, uiBytes);

	eError = psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
	                                                &psMemDesc->psMapping->sMemHandle,
	                                                psMemDesc->uiOffset,
	                                                psMemDesc->uiSize);
	if (eError != PVRSRV_OK)
	{
		goto e1;
	}

#if defined(PDUMP)
	PDUMPCOMMENT("Alloc MMU object");

	PDumpMMUMalloc(psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
	               eMMULevel,
	               &psMemDesc->sDevPAddr,
	               uiBytes,
	               uiAlign,
	               psMMUContext->psDevAttrs->eMMUType);

	PDumpMMUDumpPxEntries(eMMULevel,
	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
	                      psMemDesc->pvCpuVAddr,
	                      psMemDesc->sDevPAddr,
	                      0,
	                      uiNumEntries,
	                      NULL, NULL, 0, /* pdump symbolic info is irrelevant here */
	                      psConfig->uiBytesPerEntry,
	                      uiLog2Align,
	                      psConfig->uiAddrShift,
	                      psConfig->uiAddrMask,
	                      psConfig->uiProtMask,
	                      psConfig->uiValidEnMask,
	                      0,
	                      psMMUContext->psDevAttrs->eMMUType);
#endif

	return PVRSRV_OK;
	e1:
	_MMU_PhysMemFree(psMMUContext->psPhysMemCtx,
	                 psMemDesc);
	e0:
	PVR_ASSERT(eError != PVRSRV_OK);
	return eError;
}

/*************************************************************************/ /*!
@Function       _PxMemFree

@Description    Frees physical memory for MMU objects, de-initialises
                and PDumps it.

@Input          psMemDesc       Description of allocation

@Return         PVRSRV_OK if allocation was successful
 */
/*****************************************************************************/

static void _PxMemFree(MMU_CONTEXT *psMMUContext,
                       MMU_MEMORY_DESC *psMemDesc, MMU_LEVEL eMMULevel)
{
#if defined(MMU_CLEARMEM_ON_FREE)
	PVRSRV_ERROR eError;

	/*
		Clear the MMU object
		Note: if any MMUs are cleared with non-zero values then will need a
		custom clear function
		Note: 'Cached' is wrong for the LMA + ARM64 combination, but this is
		unlikely
	 */
	OSCachedMemSet(psMemDesc->pvCpuVAddr, 0, psMemDesc->ui32Bytes);

#if defined(PDUMP)
	PDUMPCOMMENT("Clear MMU object before freeing it");
#endif
#endif/* MMU_CLEARMEM_ON_FREE */

#if defined(PDUMP)
	PDUMPCOMMENT("Free MMU object");
	{
		PDumpMMUFree(psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
		             eMMULevel,
		             &psMemDesc->sDevPAddr,
		             psMMUContext->psDevAttrs->eMMUType);
	}
#else
	PVR_UNREFERENCED_PARAMETER(eMMULevel);
#endif
	/* free the PC */
	_MMU_PhysMemFree(psMMUContext->psPhysMemCtx, psMemDesc);
}

static INLINE PVRSRV_ERROR _SetupPTE(MMU_CONTEXT *psMMUContext,
                                     MMU_Levelx_INFO *psLevel,
                                     IMG_UINT32 uiIndex,
                                     const MMU_PxE_CONFIG *psConfig,
                                     const IMG_DEV_PHYADDR *psDevPAddr,
                                     IMG_BOOL bUnmap,
#if defined(PDUMP)
                                     const IMG_CHAR *pszMemspaceName,
                                     const IMG_CHAR *pszSymbolicAddr,
                                     IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset,
#endif
                                     IMG_UINT64 uiProtFlags)
{
	MMU_MEMORY_DESC *psMemDesc = &psLevel->sMemDesc;
	IMG_UINT64 ui64PxE64;
	IMG_UINT64 uiAddr = psDevPAddr->uiAddr;

	if (PVRSRV_IS_FEATURE_SUPPORTED(psMMUContext->psDevNode, MIPS))
	{
		/*
		 * If mapping for the MIPS FW context, check for sensitive PAs
		 */
		if (psMMUContext->psDevAttrs == psMMUContext->psDevNode->psFirmwareMMUDevAttrs)
		{
			PVRSRV_RGXDEV_INFO *psDevice = (PVRSRV_RGXDEV_INFO *)psMMUContext->psDevNode->pvDevice;

			if (RGXMIPSFW_SENSITIVE_ADDR(uiAddr))
			{
				uiAddr = psDevice->psTrampoline->sPhysAddr.uiAddr + RGXMIPSFW_TRAMPOLINE_OFFSET(uiAddr);
			}
			/* FIX_HW_BRN_63553 is mainlined for all MIPS cores */
			else if (uiAddr == 0x0 && !psDevice->sLayerParams.bDevicePA0IsValid)
			{
				PVR_DPF((PVR_DBG_ERROR, "%s attempt to map addr 0x0 in the FW but 0x0 is not considered valid.", __func__));
				return PVRSRV_ERROR_MMU_FAILED_TO_MAP_PAGE_TABLE;
			}
		}
	}

	/* Calculate Entry */
	ui64PxE64 =    uiAddr /* Calculate the offset to that base */
			>> psConfig->uiAddrLog2Align /* Shift away the useless bits, because the alignment is very coarse and we address by alignment */
			<< psConfig->uiAddrShift /* Shift back to fit address in the Px entry */
			& psConfig->uiAddrMask; /* Delete unused bits */
	ui64PxE64 |= uiProtFlags;

	/* Set the entry */
	if (psConfig->uiBytesPerEntry == 8)
	{
		IMG_UINT64 *pui64Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */

		pui64Px[uiIndex] = ui64PxE64;
	}
	else if (psConfig->uiBytesPerEntry == 4)
	{
		IMG_UINT32 *pui32Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */

		/* assert that the result fits into 32 bits before writing
		   it into the 32-bit array with a cast */
		PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));

		pui32Px[uiIndex] = (IMG_UINT32) ui64PxE64;
	}
	else
	{
		return PVRSRV_ERROR_MMU_CONFIG_IS_WRONG;
	}


	/* Log modification */
	HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
	        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
	        uiIndex, MMU_LEVEL_1,
	        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
	        !bUnmap);

#if defined (PDUMP)
	PDumpMMUDumpPxEntries(MMU_LEVEL_1,
	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
	                      psMemDesc->pvCpuVAddr,
	                      psMemDesc->sDevPAddr,
	                      uiIndex,
	                      1,
	                      pszMemspaceName,
	                      pszSymbolicAddr,
	                      uiSymbolicAddrOffset,
	                      psConfig->uiBytesPerEntry,
	                      psConfig->uiAddrLog2Align,
	                      psConfig->uiAddrShift,
	                      psConfig->uiAddrMask,
	                      psConfig->uiProtMask,
	                      psConfig->uiValidEnMask,
	                      0,
	                      psMMUContext->psDevAttrs->eMMUType);
#endif /*PDUMP*/

	return PVRSRV_OK;
}

/*************************************************************************/ /*!
@Function       _SetupPxE

@Description    Sets up an entry of an MMU object to point to the
                provided address

@Input          psMMUContext    MMU context to operate on

@Input          psLevel         Level info for MMU object

@Input          uiIndex         Index into the MMU object to setup

@Input          psConfig        MMU Px config

@Input          eMMULevel       Level of MMU object

@Input          psDevPAddr      Address to setup the MMU object to point to

@Input          pszMemspaceName Name of the PDump memory space that the entry
                                will point to

@Input          pszSymbolicAddr PDump symbolic address that the entry will
                                point to

@Input          uiProtFlags     MMU protection flags

@Return         PVRSRV_OK if the setup was successful
 */
/*****************************************************************************/
static PVRSRV_ERROR _SetupPxE(MMU_CONTEXT *psMMUContext,
                              MMU_Levelx_INFO *psLevel,
                              IMG_UINT32 uiIndex,
                              const MMU_PxE_CONFIG *psConfig,
                              MMU_LEVEL eMMULevel,
                              const IMG_DEV_PHYADDR *psDevPAddr,
#if defined(PDUMP)
                              const IMG_CHAR *pszMemspaceName,
                              const IMG_CHAR *pszSymbolicAddr,
                              IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset,
#endif
                              MMU_PROTFLAGS_T uiProtFlags,
                              IMG_UINT32 uiLog2DataPageSize)
{
	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psDevNode;
	MMU_MEMORY_DESC *psMemDesc = &psLevel->sMemDesc;

	IMG_UINT32 (*pfnDerivePxEProt4)(IMG_UINT32);
	IMG_UINT64 (*pfnDerivePxEProt8)(IMG_UINT32, IMG_UINT32);

	if (!psDevPAddr)
	{
		/* Invalidate entry */
		if (~uiProtFlags & MMU_PROTFLAGS_INVALID)
		{
			PVR_DPF((PVR_DBG_ERROR, "Error, no physical address specified, but not invalidating entry"));
			uiProtFlags |= MMU_PROTFLAGS_INVALID;
		}
		psDevPAddr = &gsBadDevPhyAddr;
	}
	else
	{
		if (uiProtFlags & MMU_PROTFLAGS_INVALID)
		{
			PVR_DPF((PVR_DBG_ERROR, "A physical address was specified when requesting invalidation of entry"));
			uiProtFlags |= MMU_PROTFLAGS_INVALID;
		}
	}

	switch (eMMULevel)
	{
		case MMU_LEVEL_3:
			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePCEProt4;
			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePCEProt8;
			break;

		case MMU_LEVEL_2:
			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePDEProt4;
			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePDEProt8;
			break;

		case MMU_LEVEL_1:
			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePTEProt4;
			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePTEProt8;
			break;

		default:
			PVR_DPF((PVR_DBG_ERROR, "%s: invalid MMU level", __func__));
			return PVRSRV_ERROR_INVALID_PARAMS;
	}

	/* How big is a PxE in bytes? */
	/* Filling the actual Px entry with an address */
	switch (psConfig->uiBytesPerEntry)
	{
		case 4:
		{
			IMG_UINT32 *pui32Px;
			IMG_UINT64 ui64PxE64;

			pui32Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */

			ui64PxE64 = psDevPAddr->uiAddr               /* Calculate the offset to that base */
					>> psConfig->uiAddrLog2Align /* Shift away the unnecessary bits of the address */
					<< psConfig->uiAddrShift     /* Shift back to fit address in the Px entry */
					& psConfig->uiAddrMask;      /* Delete unused higher bits */

			ui64PxE64 |= (IMG_UINT64)pfnDerivePxEProt4(uiProtFlags);
			/* assert that the result fits into 32 bits before writing
			   it into the 32-bit array with a cast */
			PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));

			/* We should never invalidate an invalid page */
			if (uiProtFlags & MMU_PROTFLAGS_INVALID)
			{
				PVR_ASSERT(pui32Px[uiIndex] != ui64PxE64);
			}
			pui32Px[uiIndex] = (IMG_UINT32) ui64PxE64;
			HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
			        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
			        uiIndex, eMMULevel,
			        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
			        (uiProtFlags & MMU_PROTFLAGS_INVALID)? 0: 1);
			break;
		}
		case 8:
		{
			IMG_UINT64 *pui64Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */

			pui64Px[uiIndex] = psDevPAddr->uiAddr             /* Calculate the offset to that base */
					>> psConfig->uiAddrLog2Align  /* Shift away the unnecessary bits of the address */
					<< psConfig->uiAddrShift      /* Shift back to fit address in the Px entry */
					& psConfig->uiAddrMask;       /* Delete unused higher bits */
			pui64Px[uiIndex] |= pfnDerivePxEProt8(uiProtFlags, uiLog2DataPageSize);

			HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
			        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
			        uiIndex, eMMULevel,
			        HTBLOG_U64_BITS_HIGH(pui64Px[uiIndex]), HTBLOG_U64_BITS_LOW(pui64Px[uiIndex]),
			        (uiProtFlags & MMU_PROTFLAGS_INVALID)? 0: 1);
			break;
		}
		default:
			PVR_DPF((PVR_DBG_ERROR, "%s: PxE size not supported (%d) for level %d",
					__func__, psConfig->uiBytesPerEntry, eMMULevel));

			return PVRSRV_ERROR_MMU_CONFIG_IS_WRONG;
	}

#if defined (PDUMP)
	PDumpMMUDumpPxEntries(eMMULevel,
	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
	                      psMemDesc->pvCpuVAddr,
	                      psMemDesc->sDevPAddr,
	                      uiIndex,
	                      1,
	                      pszMemspaceName,
	                      pszSymbolicAddr,
	                      uiSymbolicAddrOffset,
	                      psConfig->uiBytesPerEntry,
	                      psConfig->uiAddrLog2Align,
	                      psConfig->uiAddrShift,
	                      psConfig->uiAddrMask,
	                      psConfig->uiProtMask,
	                      psConfig->uiValidEnMask,
	                      0,
	                      psMMUContext->psDevAttrs->eMMUType);
#endif

	psDevNode->pfnMMUCacheInvalidate(psDevNode, psMMUContext->hDevData,
	                                 eMMULevel,
	                                 (uiProtFlags & MMU_PROTFLAGS_INVALID)?IMG_TRUE:IMG_FALSE);

	return PVRSRV_OK;
}

/*****************************************************************************
 *                   MMU host control functions (Level Info)                 *
 *****************************************************************************/


/*************************************************************************/ /*!
@Function       _MMU_FreeLevel

@Description    Recursively frees the specified range of Px entries. If any
                level has its last reference dropped then the MMU object
                memory and the MMU_Levelx_Info will be freed.

				At each level we might be crossing a boundary from one Px to
				another. The values for auiStartArray should be by used for
				the first call into each level and the values in auiEndArray
				should only be used in the last call for each level.
				In order to determine if this is the first/last call we pass
				in bFirst and bLast.
				When one level calls down to the next only if bFirst/bLast is set
				and it's the first/last iteration of the loop at its level will
				bFirst/bLast set for the next recursion.
				This means that each iteration has the knowledge of the previous
				level which is required.

@Input          psMMUContext    MMU context to operate on

@Input          psLevel                 Level info on which to free the
                                        specified range

@Input          auiStartArray           Array of start indexes (one for each level)

@Input          auiEndArray             Array of end indexes (one for each level)

@Input          auiEntriesPerPxArray    Array of number of entries for the Px
                                        (one for each level)

@Input          apsConfig               Array of PxE configs (one for each level)

@Input          aeMMULevel              Array of MMU levels (one for each level)

@Input          pui32CurrentLevel       Pointer to a variable which is set to our
                                        current level

@Input          uiStartIndex            Start index of the range to free

@Input          uiEndIndex              End index of the range to free

@Input			bFirst                  This is the first call for this level

@Input			bLast                   This is the last call for this level

@Return         IMG_TRUE if the last reference to psLevel was dropped
 */
/*****************************************************************************/
static IMG_BOOL _MMU_FreeLevel(MMU_CONTEXT *psMMUContext,
                               MMU_Levelx_INFO *psLevel,
                               IMG_UINT32 auiStartArray[],
                               IMG_UINT32 auiEndArray[],
                               IMG_UINT32 auiEntriesPerPxArray[],
                               const MMU_PxE_CONFIG *apsConfig[],
                               MMU_LEVEL aeMMULevel[],
                               IMG_UINT32 *pui32CurrentLevel,
                               IMG_UINT32 uiStartIndex,
                               IMG_UINT32 uiEndIndex,
                               IMG_BOOL bFirst,
                               IMG_BOOL bLast,
                               IMG_UINT32 uiLog2DataPageSize)
{
	IMG_UINT32 uiThisLevel = *pui32CurrentLevel;
	const MMU_PxE_CONFIG *psConfig = apsConfig[uiThisLevel];
	IMG_UINT32 i;
	IMG_BOOL bFreed = IMG_FALSE;

	/* Sanity check */
	PVR_ASSERT(*pui32CurrentLevel < MMU_MAX_LEVEL);
	PVR_ASSERT(psLevel != NULL);

	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_FreeLevel: level = %d, range %d - %d, refcount = %d",
			aeMMULevel[uiThisLevel], uiStartIndex,
			uiEndIndex, psLevel->ui32RefCount));

	for (i = uiStartIndex;(i < uiEndIndex) && (psLevel != NULL);i++)
	{
		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
		{
			MMU_Levelx_INFO *psNextLevel = psLevel->apsNextLevel[i];
			IMG_UINT32 uiNextStartIndex;
			IMG_UINT32 uiNextEndIndex;
			IMG_BOOL bNextFirst;
			IMG_BOOL bNextLast;

			/* If we're crossing a Px then the start index changes */
			if (bFirst && (i == uiStartIndex))
			{
				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
				bNextFirst = IMG_TRUE;
			}
			else
			{
				uiNextStartIndex = 0;
				bNextFirst = IMG_FALSE;
			}

			/* If we're crossing a Px then the end index changes */
			if (bLast && (i == (uiEndIndex - 1)))
			{
				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
				bNextLast = IMG_TRUE;
			}
			else
			{
				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
				bNextLast = IMG_FALSE;
			}

			/* Recurse into the next level */
			(*pui32CurrentLevel)++;
			if (_MMU_FreeLevel(psMMUContext, psNextLevel, auiStartArray,
			                   auiEndArray, auiEntriesPerPxArray,
			                   apsConfig, aeMMULevel, pui32CurrentLevel,
			                   uiNextStartIndex, uiNextEndIndex,
			                   bNextFirst, bNextLast, uiLog2DataPageSize))
			{
				PVRSRV_ERROR eError;

				/* Un-wire the entry */
				eError = _SetupPxE(psMMUContext,
				                   psLevel,
				                   i,
				                   psConfig,
				                   aeMMULevel[uiThisLevel],
				                   NULL,
#if defined(PDUMP)
				                   NULL,	/* Only required for data page */
				                   NULL,	/* Only required for data page */
				                   0,      /* Only required for data page */
#endif
				                   MMU_PROTFLAGS_INVALID,
				                   uiLog2DataPageSize);

				PVR_ASSERT(eError == PVRSRV_OK);

				/* Free table of the level below, pointed to by this table entry.
				 * We don't destroy the table inside the above _MMU_FreeLevel call because we
				 * first have to set the table entry of the level above to invalid. */
				_PxMemFree(psMMUContext, &psNextLevel->sMemDesc, aeMMULevel[*pui32CurrentLevel]);
				OSFreeMem(psNextLevel);

				/* The level below us is empty, drop the refcount and clear the pointer */
				psLevel->ui32RefCount--;
				psLevel->apsNextLevel[i] = NULL;

				/* Check we haven't wrapped around */
				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
			}
			(*pui32CurrentLevel)--;
		}
		else
		{
			psLevel->ui32RefCount--;
		}

		/*
		   Free this level if it is no longer referenced, unless it's the base
		   level in which case it's part of the MMU context and should be freed
		   when the MMU context is freed
		 */
		if ((psLevel->ui32RefCount == 0) && (psLevel != &psMMUContext->sBaseLevelInfo))
		{
			bFreed = IMG_TRUE;
		}
	}

	/* Level one flushing is done when we actually write the table entries */
	if ((aeMMULevel[uiThisLevel] != MMU_LEVEL_1) && (psLevel != NULL))
	{
		psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
		                                       &psLevel->sMemDesc.psMapping->sMemHandle,
		                                       uiStartIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
		                                       (uiEndIndex - uiStartIndex) * psConfig->uiBytesPerEntry);
	}

	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_FreeLevel end: level = %d, refcount = %d",
			aeMMULevel[uiThisLevel], bFreed?0: (psLevel)?psLevel->ui32RefCount:-1));

	return bFreed;
}

/*************************************************************************/ /*!
@Function       _MMU_AllocLevel

@Description    Recursively allocates the specified range of Px entries. If any
                level has its last reference dropped then the MMU object
                memory and the MMU_Levelx_Info will be freed.

				At each level we might be crossing a boundary from one Px to
				another. The values for auiStartArray should be by used for
				the first call into each level and the values in auiEndArray
				should only be used in the last call for each level.
				In order to determine if this is the first/last call we pass
				in bFirst and bLast.
				When one level calls down to the next only if bFirst/bLast is set
				and it's the first/last iteration of the loop at its level will
				bFirst/bLast set for the next recursion.
				This means that each iteration has the knowledge of the previous
				level which is required.

@Input          psMMUContext    MMU context to operate on

@Input          psLevel                 Level info on which to to free the
                                        specified range

@Input          auiStartArray           Array of start indexes (one for each level)

@Input          auiEndArray             Array of end indexes (one for each level)

@Input          auiEntriesPerPxArray    Array of number of entries for the Px
                                        (one for each level)

@Input          apsConfig               Array of PxE configs (one for each level)

@Input          aeMMULevel              Array of MMU levels (one for each level)

@Input          pui32CurrentLevel       Pointer to a variable which is set to our
                                        current level

@Input          uiStartIndex            Start index of the range to free

@Input          uiEndIndex              End index of the range to free

@Input			bFirst                  This is the first call for this level

@Input			bLast                   This is the last call for this level

@Return         IMG_TRUE if the last reference to psLevel was dropped
 */
/*****************************************************************************/
static PVRSRV_ERROR _MMU_AllocLevel(MMU_CONTEXT *psMMUContext,
                                    MMU_Levelx_INFO *psLevel,
                                    IMG_UINT32 auiStartArray[],
                                    IMG_UINT32 auiEndArray[],
                                    IMG_UINT32 auiEntriesPerPxArray[],
                                    const MMU_PxE_CONFIG *apsConfig[],
                                    MMU_LEVEL aeMMULevel[],
                                    IMG_UINT32 *pui32CurrentLevel,
                                    IMG_UINT32 uiStartIndex,
                                    IMG_UINT32 uiEndIndex,
                                    IMG_BOOL bFirst,
                                    IMG_BOOL bLast,
                                    IMG_UINT32 uiLog2DataPageSize)
{
	IMG_UINT32 uiThisLevel = *pui32CurrentLevel; /* Starting with 0 */
	const MMU_PxE_CONFIG *psConfig = apsConfig[uiThisLevel]; /* The table config for the current level */
	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
	IMG_UINT32 uiAllocState = 99; /* Debug info to check what progress was made in the function. Updated during this function. */
	IMG_UINT32 i;

	/* Sanity check */
	PVR_ASSERT(*pui32CurrentLevel < MMU_MAX_LEVEL);

	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_AllocLevel: level = %d, range %d - %d, refcount = %d",
			aeMMULevel[uiThisLevel], uiStartIndex,
			uiEndIndex, psLevel->ui32RefCount));

	/* Go from uiStartIndex to uiEndIndex through the Px */
	for (i = uiStartIndex;i < uiEndIndex;i++)
	{
		/* Only try an allocation if this is not the last level */
		/*Because a PT allocation is already done while setting the entry in PD */
		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
		{
			IMG_UINT32 uiNextStartIndex;
			IMG_UINT32 uiNextEndIndex;
			IMG_BOOL bNextFirst;
			IMG_BOOL bNextLast;

			/* If there is already a next Px level existing, do not allocate it */
			if (!psLevel->apsNextLevel[i])
			{
				MMU_Levelx_INFO *psNextLevel;
				IMG_UINT32 ui32AllocSize;
				IMG_UINT32 uiNextEntries;

				/* Allocate and setup the next level */
				uiNextEntries = auiEntriesPerPxArray[uiThisLevel + 1];
				ui32AllocSize = sizeof(MMU_Levelx_INFO);
				if (aeMMULevel[uiThisLevel + 1] != MMU_LEVEL_1)
				{
					ui32AllocSize += sizeof(MMU_Levelx_INFO *) * (uiNextEntries - 1);
				}
				psNextLevel = OSAllocZMem(ui32AllocSize);
				if (psNextLevel == NULL)
				{
					uiAllocState = 0;
					goto e0;
				}

				/* Hook in this level for next time */
				psLevel->apsNextLevel[i] = psNextLevel;

				psNextLevel->ui32NumOfEntries = uiNextEntries;
				psNextLevel->ui32RefCount = 0;
				/* Allocate Px memory for a sub level*/
				eError = _PxMemAlloc(psMMUContext, uiNextEntries, apsConfig[uiThisLevel + 1],
				                     aeMMULevel[uiThisLevel + 1],
				                     &psNextLevel->sMemDesc,
				                     psConfig->uiAddrLog2Align);
				if (eError != PVRSRV_OK)
				{
					uiAllocState = 1;
					goto e0;
				}

				/* Wire up the entry */
				eError = _SetupPxE(psMMUContext,
				                   psLevel,
				                   i,
				                   psConfig,
				                   aeMMULevel[uiThisLevel],
				                   &psNextLevel->sMemDesc.sDevPAddr,
#if defined(PDUMP)
				                   NULL, /* Only required for data page */
				                   NULL, /* Only required for data page */
				                   0,    /* Only required for data page */
#endif
				                   0,
				                   uiLog2DataPageSize);

				if (eError != PVRSRV_OK)
				{
					uiAllocState = 2;
					goto e0;
				}

				psLevel->ui32RefCount++;
			}

			/* If we're crossing a Px then the start index changes */
			if (bFirst && (i == uiStartIndex))
			{
				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
				bNextFirst = IMG_TRUE;
			}
			else
			{
				uiNextStartIndex = 0;
				bNextFirst = IMG_FALSE;
			}

			/* If we're crossing a Px then the end index changes */
			if (bLast && (i == (uiEndIndex - 1)))
			{
				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
				bNextLast = IMG_TRUE;
			}
			else
			{
				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
				bNextLast = IMG_FALSE;
			}

			/* Recurse into the next level */
			(*pui32CurrentLevel)++;
			eError = _MMU_AllocLevel(psMMUContext, psLevel->apsNextLevel[i],
			                         auiStartArray,
			                         auiEndArray,
			                         auiEntriesPerPxArray,
			                         apsConfig,
			                         aeMMULevel,
			                         pui32CurrentLevel,
			                         uiNextStartIndex,
			                         uiNextEndIndex,
			                         bNextFirst,
			                         bNextLast,
			                         uiLog2DataPageSize);
			(*pui32CurrentLevel)--;
			if (eError != PVRSRV_OK)
			{
				uiAllocState = 2;
				goto e0;
			}
		}
		else
		{
			/* All we need to do for level 1 is bump the refcount */
			psLevel->ui32RefCount++;
		}
		PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
	}

	/* Level one flushing is done when we actually write the table entries */
	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
	{
		eError = psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
		                                                &psLevel->sMemDesc.psMapping->sMemHandle,
		                                                uiStartIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
		                                                (uiEndIndex - uiStartIndex) * psConfig->uiBytesPerEntry);
		if (eError != PVRSRV_OK)
			goto e0;
	}

	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_AllocLevel end: level = %d, refcount = %d",
			aeMMULevel[uiThisLevel], psLevel->ui32RefCount));
	return PVRSRV_OK;

	e0:
	/* Sanity check that we've not come down this route unexpectedly */
	PVR_ASSERT(uiAllocState!=99);
	PVR_DPF((PVR_DBG_ERROR, "_MMU_AllocLevel: Error %d allocating Px for level %d in stage %d"
			,eError, aeMMULevel[uiThisLevel], uiAllocState));

	/* the start value of index variable i is nor initialised on purpose
	   indeed this for loop deinitialise what has already been initialised
	   just before failing in reverse order. So the i index has already the
	   right value. */
	for (/* i already set */; i>= uiStartIndex && i< uiEndIndex; i--)
	{
		switch (uiAllocState)
		{
			IMG_UINT32 uiNextStartIndex;
			IMG_UINT32 uiNextEndIndex;
			IMG_BOOL bNextFirst;
			IMG_BOOL bNextLast;

			case 3:
				/* If we're crossing a Px then the start index changes */
				if (bFirst && (i == uiStartIndex))
				{
					uiNextStartIndex = auiStartArray[uiThisLevel + 1];
					bNextFirst = IMG_TRUE;
				}
				else
				{
					uiNextStartIndex = 0;
					bNextFirst = IMG_FALSE;
				}

				/* If we're crossing a Px then the end index changes */
				if (bLast && (i == (uiEndIndex - 1)))
				{
					uiNextEndIndex = auiEndArray[uiThisLevel + 1];
					bNextLast = IMG_TRUE;
				}
				else
				{
					uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
					bNextLast = IMG_FALSE;
				}

				if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
				{
					(*pui32CurrentLevel)++;
					if (_MMU_FreeLevel(psMMUContext, psLevel->apsNextLevel[i],
					                   auiStartArray, auiEndArray,
					                   auiEntriesPerPxArray, apsConfig,
					                   aeMMULevel, pui32CurrentLevel,
					                   uiNextStartIndex, uiNextEndIndex,
					                   bNextFirst, bNextLast, uiLog2DataPageSize))
					{
						psLevel->ui32RefCount--;
						psLevel->apsNextLevel[i] = NULL;

						/* Check we haven't wrapped around */
						PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
					}
					(*pui32CurrentLevel)--;
				}
				else
				{
					/* We should never come down this path, but it's here
						   for completeness */
					psLevel->ui32RefCount--;

					/* Check we haven't wrapped around */
					PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
				}

				__fallthrough;
			case 2:
				if (psLevel->apsNextLevel[i] != NULL  &&
						psLevel->apsNextLevel[i]->ui32RefCount == 0)
				{
					_PxMemFree(psMMUContext, &psLevel->sMemDesc,
					           aeMMULevel[uiThisLevel]);
				}

				__fallthrough;
			case 1:
				if (psLevel->apsNextLevel[i] != NULL  &&
						psLevel->apsNextLevel[i]->ui32RefCount == 0)
				{
					OSFreeMem(psLevel->apsNextLevel[i]);
					psLevel->apsNextLevel[i] = NULL;
				}

				__fallthrough;
			case 0:
				uiAllocState = 3;
				break;
		}
	}
	return eError;
}

/*****************************************************************************
 *                   MMU page table functions                                *
 *****************************************************************************/

/*************************************************************************/ /*!
@Function       _MMU_GetLevelData

@Description    Get the all the level data and calculates the indexes for the
                specified address range

@Input          psMMUContext            MMU context to operate on

@Input          sDevVAddrStart          Start device virtual address

@Input          sDevVAddrEnd            End device virtual address

@Input          uiLog2DataPageSize      Log2 of the page size to use

@Input          auiStartArray           Array of start indexes (one for each level)

@Input          auiEndArray             Array of end indexes (one for each level)

@Input          uiEntriesPerPxArray     Array of number of entries for the Px
                                        (one for each level)

@Input          apsConfig               Array of PxE configs (one for each level)

@Input          aeMMULevel              Array of MMU levels (one for each level)

@Input          ppsMMUDevVAddrConfig    Device virtual address config

@Input			phPriv					Private data of page size config

@Return         IMG_TRUE if the last reference to psLevel was dropped
 */
/*****************************************************************************/
static void _MMU_GetLevelData(MMU_CONTEXT *psMMUContext,
                              IMG_DEV_VIRTADDR sDevVAddrStart,
                              IMG_DEV_VIRTADDR sDevVAddrEnd,
                              IMG_UINT32 uiLog2DataPageSize,
                              IMG_UINT32 auiStartArray[],
                              IMG_UINT32 auiEndArray[],
                              IMG_UINT32 auiEntriesPerPx[],
                              const MMU_PxE_CONFIG *apsConfig[],
                              MMU_LEVEL aeMMULevel[],
                              const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
                              IMG_HANDLE *phPriv)
{
	const MMU_PxE_CONFIG *psMMUPDEConfig;
	const MMU_PxE_CONFIG *psMMUPTEConfig;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
	PVRSRV_ERROR eError;
	IMG_UINT32 i = 0;

	eError = psDevAttrs->pfnGetPageSizeConfiguration(uiLog2DataPageSize,
	                                                 &psMMUPDEConfig,
	                                                 &psMMUPTEConfig,
	                                                 ppsMMUDevVAddrConfig,
	                                                 phPriv);
	PVR_ASSERT(eError == PVRSRV_OK);

	psDevVAddrConfig = *ppsMMUDevVAddrConfig;

	if (psDevVAddrConfig->uiPCIndexMask != 0)
	{
		auiStartArray[i] = _CalcPCEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
		auiEndArray[i] = _CalcPCEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
		auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPC;
		apsConfig[i] = psDevAttrs->psBaseConfig;
		aeMMULevel[i] = MMU_LEVEL_3;
		i++;
	}

	if (psDevVAddrConfig->uiPDIndexMask != 0)
	{
		auiStartArray[i] = _CalcPDEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
		auiEndArray[i] = _CalcPDEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
		auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPD;
		if (i == 0)
		{
			apsConfig[i] = psDevAttrs->psBaseConfig;
		}
		else
		{
			apsConfig[i] = psMMUPDEConfig;
		}
		aeMMULevel[i] = MMU_LEVEL_2;
		i++;
	}

	/*
		There is always a PTE entry so we have a slightly different behaviour than above.
		E.g. for 2 MB RGX pages the uiPTIndexMask is 0x0000000000 but still there
		is a PT with one entry.

	 */
	auiStartArray[i] = _CalcPTEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
	if (psDevVAddrConfig->uiPTIndexMask !=0)
	{
		auiEndArray[i] = _CalcPTEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
	}
	else
	{
		/*
			If the PTE mask is zero it means there is only 1 PTE and thus, as an
			an exclusive bound, the end array index is equal to the start index + 1.
		 */

		auiEndArray[i] = auiStartArray[i] + 1;
	}

	auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPT;

	if (i == 0)
	{
		apsConfig[i] = psDevAttrs->psBaseConfig;
	}
	else
	{
		apsConfig[i] = psMMUPTEConfig;
	}
	aeMMULevel[i] = MMU_LEVEL_1;
}

static void _MMU_PutLevelData(MMU_CONTEXT *psMMUContext, IMG_HANDLE hPriv)
{
	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;

	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
}

/*************************************************************************/ /*!
@Function       _AllocPageTables

@Description    Allocate page tables and any higher level MMU objects required
                for the specified virtual range

@Input          psMMUContext            MMU context to operate on

@Input          sDevVAddrStart          Start device virtual address

@Input          sDevVAddrEnd            End device virtual address

@Input          uiLog2DataPageSize      Page size of the data pages

@Return         PVRSRV_OK if the allocation was successful
 */
/*****************************************************************************/
static PVRSRV_ERROR
_AllocPageTables(MMU_CONTEXT *psMMUContext,
                 IMG_DEV_VIRTADDR sDevVAddrStart,
                 IMG_DEV_VIRTADDR sDevVAddrEnd,
                 IMG_UINT32 uiLog2DataPageSize)
{
	PVRSRV_ERROR eError;
	IMG_UINT32 auiStartArray[MMU_MAX_LEVEL];
	IMG_UINT32 auiEndArray[MMU_MAX_LEVEL];
	IMG_UINT32 auiEntriesPerPx[MMU_MAX_LEVEL];
	MMU_LEVEL aeMMULevel[MMU_MAX_LEVEL];
	const MMU_PxE_CONFIG *apsConfig[MMU_MAX_LEVEL];
	const MMU_DEVVADDR_CONFIG	*psDevVAddrConfig;
	IMG_HANDLE hPriv;
	IMG_UINT32 ui32CurrentLevel = 0;

	PVR_DPF((PVR_DBG_ALLOC,
			"_AllocPageTables: vaddr range: "IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
			sDevVAddrStart.uiAddr,
			sDevVAddrEnd.uiAddr
	));

#if defined(PDUMP)
	PDUMPCOMMENT("Allocating page tables for %"IMG_UINT64_FMTSPEC" bytes virtual range: "
	             IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
	             (IMG_UINT64)sDevVAddrEnd.uiAddr - (IMG_UINT64)sDevVAddrStart.uiAddr,
	             (IMG_UINT64)sDevVAddrStart.uiAddr,
	             (IMG_UINT64)sDevVAddrEnd.uiAddr);
#endif

	_MMU_GetLevelData(psMMUContext, sDevVAddrStart, sDevVAddrEnd,
	                  (IMG_UINT32) uiLog2DataPageSize, auiStartArray, auiEndArray,
	                  auiEntriesPerPx, apsConfig, aeMMULevel,
	                  &psDevVAddrConfig, &hPriv);

	HTBLOGK(HTB_SF_MMU_PAGE_OP_ALLOC,
	        HTBLOG_U64_BITS_HIGH(sDevVAddrStart.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrStart.uiAddr),
	        HTBLOG_U64_BITS_HIGH(sDevVAddrEnd.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrEnd.uiAddr));

	eError = _MMU_AllocLevel(psMMUContext, &psMMUContext->sBaseLevelInfo,
	                         auiStartArray, auiEndArray, auiEntriesPerPx,
	                         apsConfig, aeMMULevel, &ui32CurrentLevel,
	                         auiStartArray[0], auiEndArray[0],
	                         IMG_TRUE, IMG_TRUE, uiLog2DataPageSize);

	_MMU_PutLevelData(psMMUContext, hPriv);

	return eError;
}

/*************************************************************************/ /*!
@Function       _FreePageTables

@Description    Free page tables and any higher level MMU objects at are no
                longer referenced for the specified virtual range.
                This will fill the temporary free list of the MMU context which
                needs cleanup after the call.

@Input          psMMUContext            MMU context to operate on

@Input          sDevVAddrStart          Start device virtual address

@Input          sDevVAddrEnd            End device virtual address

@Input          uiLog2DataPageSize      Page size of the data pages

@Return         None
 */
/*****************************************************************************/
static void _FreePageTables(MMU_CONTEXT *psMMUContext,
                            IMG_DEV_VIRTADDR sDevVAddrStart,
                            IMG_DEV_VIRTADDR sDevVAddrEnd,
                            IMG_UINT32 uiLog2DataPageSize)
{
	IMG_UINT32 auiStartArray[MMU_MAX_LEVEL];
	IMG_UINT32 auiEndArray[MMU_MAX_LEVEL];
	IMG_UINT32 auiEntriesPerPx[MMU_MAX_LEVEL];
	MMU_LEVEL aeMMULevel[MMU_MAX_LEVEL];
	const MMU_PxE_CONFIG *apsConfig[MMU_MAX_LEVEL];
	const MMU_DEVVADDR_CONFIG	*psDevVAddrConfig;
	IMG_UINT32 ui32CurrentLevel = 0;
	IMG_HANDLE hPriv;

	PVR_DPF((PVR_DBG_ALLOC,
			"_FreePageTables: vaddr range: "IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
			sDevVAddrStart.uiAddr,
			sDevVAddrEnd.uiAddr
	));

	_MMU_GetLevelData(psMMUContext, sDevVAddrStart, sDevVAddrEnd,
	                  uiLog2DataPageSize, auiStartArray, auiEndArray,
	                  auiEntriesPerPx, apsConfig, aeMMULevel,
	                  &psDevVAddrConfig, &hPriv);

	HTBLOGK(HTB_SF_MMU_PAGE_OP_FREE,
	        HTBLOG_U64_BITS_HIGH(sDevVAddrStart.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrStart.uiAddr),
	        HTBLOG_U64_BITS_HIGH(sDevVAddrEnd.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrEnd.uiAddr));

	/* ignoring return code, in this case there should be no references
	 * to the level anymore, and at this stage there is nothing to do with
	 * the return status */
	(void) _MMU_FreeLevel(psMMUContext, &psMMUContext->sBaseLevelInfo,
	                      auiStartArray, auiEndArray, auiEntriesPerPx,
	                      apsConfig, aeMMULevel, &ui32CurrentLevel,
	                      auiStartArray[0], auiEndArray[0],
	                      IMG_TRUE, IMG_TRUE, uiLog2DataPageSize);

	_MMU_PutLevelData(psMMUContext, hPriv);
}


/*************************************************************************/ /*!
@Function       _MMU_GetPTInfo

@Description    Get the PT level information and PT entry index for the specified
                virtual address

@Input          psMMUContext            MMU context to operate on

@Input          psDevVAddr              Device virtual address to get the PTE info
                                        from.

@Input          psDevVAddrConfig        The current virtual address config obtained
                                        by another function call before.

@Output         psLevel                 Level info of the PT

@Output         pui32PTEIndex           Index into the PT the address corresponds to

@Return         None
 */
/*****************************************************************************/
static INLINE void _MMU_GetPTInfo(MMU_CONTEXT                *psMMUContext,
                                  IMG_DEV_VIRTADDR            sDevVAddr,
                                  const MMU_DEVVADDR_CONFIG  *psDevVAddrConfig,
                                  MMU_Levelx_INFO           **psLevel,
                                  IMG_UINT32                 *pui32PTEIndex)
{
	MMU_Levelx_INFO *psLocalLevel = NULL;
	MMU_LEVEL eMMULevel = psMMUContext->psDevAttrs->eTopLevel;
	IMG_UINT32 uiPCEIndex;
	IMG_UINT32 uiPDEIndex;

	if ((eMMULevel <= MMU_LEVEL_0) || (eMMULevel >= MMU_LEVEL_LAST))
	{
		PVR_DPF((PVR_DBG_ERROR, "_MMU_GetPTEInfo: Invalid MMU level"));
		psLevel = NULL;
		return;
	}

	for (; eMMULevel > MMU_LEVEL_0; eMMULevel--)
	{
		if (eMMULevel == MMU_LEVEL_3)
		{
			/* find the page directory containing the PCE */
			uiPCEIndex = _CalcPCEIdx (sDevVAddr, psDevVAddrConfig,
			                          IMG_FALSE);
			psLocalLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiPCEIndex];
		}

		if (eMMULevel == MMU_LEVEL_2)
		{
			/* find the page table containing the PDE */
			uiPDEIndex = _CalcPDEIdx (sDevVAddr, psDevVAddrConfig,
			                          IMG_FALSE);
			if (psLocalLevel != NULL)
			{
				psLocalLevel = psLocalLevel->apsNextLevel[uiPDEIndex];
			}
			else
			{
				psLocalLevel =
						psMMUContext->sBaseLevelInfo.apsNextLevel[uiPDEIndex];
			}
		}

		if (eMMULevel == MMU_LEVEL_1)
		{
			/* find PTE index into page table */
			*pui32PTEIndex = _CalcPTEIdx (sDevVAddr, psDevVAddrConfig,
			                              IMG_FALSE);
			if (psLocalLevel == NULL)
			{
				psLocalLevel = &psMMUContext->sBaseLevelInfo;
			}
		}
	}
	*psLevel = psLocalLevel;
}

/*************************************************************************/ /*!
@Function       _MMU_GetPTConfig

@Description    Get the level config. Call _MMU_PutPTConfig after use!

@Input          psMMUContext            MMU context to operate on

@Input          uiLog2DataPageSize      Log 2 of the page size

@Output         ppsConfig               Config of the PTE

@Output         phPriv                  Private data handle to be passed back
                                        when the info is put

@Output         ppsDevVAddrConfig       Config of the device virtual addresses

@Return         None
 */
/*****************************************************************************/
static INLINE void _MMU_GetPTConfig(MMU_CONTEXT               *psMMUContext,
                                    IMG_UINT32                  uiLog2DataPageSize,
                                    const MMU_PxE_CONFIG      **ppsConfig,
                                    IMG_HANDLE                 *phPriv,
                                    const MMU_DEVVADDR_CONFIG **ppsDevVAddrConfig)
{
	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
	const MMU_PxE_CONFIG *psPDEConfig;
	const MMU_PxE_CONFIG *psPTEConfig;

	if (psDevAttrs->pfnGetPageSizeConfiguration(uiLog2DataPageSize,
	                                            &psPDEConfig,
	                                            &psPTEConfig,
	                                            &psDevVAddrConfig,
	                                            phPriv) != PVRSRV_OK)
	{
		/*
		   There should be no way we got here unless uiLog2DataPageSize
		   has changed after the MMU_Alloc call (in which case it's a bug in
		   the MM code)
		 */
		PVR_DPF((PVR_DBG_ERROR, "_MMU_GetPTConfig: Could not get valid page size config"));
		PVR_ASSERT(0);
	}

	*ppsConfig = psPTEConfig;
	*ppsDevVAddrConfig = psDevVAddrConfig;
}

/*************************************************************************/ /*!
@Function       _MMU_PutPTConfig

@Description    Put the level info. Has to be called after _MMU_GetPTConfig to
                ensure correct refcounting.

@Input          psMMUContext            MMU context to operate on

@Input          phPriv                  Private data handle created by
                                        _MMU_GetPTConfig.

@Return         None
 */
/*****************************************************************************/
static INLINE void _MMU_PutPTConfig(MMU_CONTEXT *psMMUContext,
                                    IMG_HANDLE hPriv)
{
	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;

	if (psDevAttrs->pfnPutPageSizeConfiguration(hPriv) != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Could not put page size config",
				__func__));
		PVR_ASSERT(0);
	}
}


/*****************************************************************************
 *                     Public interface functions                            *
 *****************************************************************************/

/*
	MMU_ContextCreate
 */
PVRSRV_ERROR
MMU_ContextCreate(PVRSRV_DEVICE_NODE *psDevNode,
                  MMU_CONTEXT **ppsMMUContext,
                  MMU_DEVICEATTRIBS *psDevAttrs)
{
	MMU_CONTEXT *psMMUContext;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
	const MMU_PxE_CONFIG *psConfig;
	MMU_PHYSMEM_CONTEXT *psCtx;
	IMG_UINT32 ui32BaseObjects;
	IMG_UINT32 ui32Size;
	IMG_CHAR sBuf[40];
	PVRSRV_ERROR eError = PVRSRV_OK;

	psConfig = psDevAttrs->psBaseConfig;
	psDevVAddrConfig = psDevAttrs->psTopLevelDevVAddrConfig;

	switch (psDevAttrs->eTopLevel)
	{
		case MMU_LEVEL_3:
			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPC;
			break;

		case MMU_LEVEL_2:
			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPD;
			break;

		case MMU_LEVEL_1:
			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPT;
			break;

		default:
			PVR_DPF((PVR_DBG_ERROR,
					"%s: Invalid MMU config", __func__));
			eError = PVRSRV_ERROR_INVALID_PARAMS;
			goto e0;
	}

	/* Allocate the MMU context with the Level 1 Px info's */
	ui32Size = sizeof(MMU_CONTEXT) +
			((ui32BaseObjects - 1) * sizeof(MMU_Levelx_INFO *));

	psMMUContext = OSAllocZMem(ui32Size);
	if (psMMUContext == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Call to OSAllocZMem failed", __func__));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e0;
	}

#if defined(PDUMP)
	/* Clear the refcount */
	psMMUContext->ui32PDumpContextIDRefCount = 0;
#endif
	/* Record Device specific attributes in the context for subsequent use */
	psMMUContext->psDevAttrs = psDevAttrs;
	psMMUContext->psDevNode = psDevNode;

#if defined(SUPPORT_GPUVIRT_VALIDATION)
	{
		IMG_UINT32 ui32OSid, ui32OSidReg;
		IMG_BOOL bOSidAxiProt;

		RetrieveOSidsfromPidList(OSGetCurrentClientProcessIDKM(), &ui32OSid, &ui32OSidReg, &bOSidAxiProt);

		MMU_SetOSids(psMMUContext, ui32OSid, ui32OSidReg, bOSidAxiProt);
	}
#endif

	/*
	  Allocate physmem context and set it up
	 */
	psCtx = OSAllocZMem(sizeof(MMU_PHYSMEM_CONTEXT));
	if (psCtx == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Call to OSAllocZMem failed", __func__));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e1;
	}
	psMMUContext->psPhysMemCtx = psCtx;

	psCtx->psDevNode = psDevNode;

	OSSNPrintf(sBuf, sizeof(sBuf), "pgtables %p", psCtx);
	psCtx->uiPhysMemRANameAllocSize = OSStringLength(sBuf)+1;
	psCtx->pszPhysMemRAName = OSAllocMem(psCtx->uiPhysMemRANameAllocSize);
	if (psCtx->pszPhysMemRAName == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR, "%s: Out of memory", __func__));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e2;
	}

	OSStringCopy(psCtx->pszPhysMemRAName, sBuf);

	psCtx->psPhysMemRA = RA_Create(psCtx->pszPhysMemRAName,
	                               /* subsequent import */
	                               psDevNode->uiMMUPxLog2AllocGran,
	                               RA_LOCKCLASS_1,
	                               _MMU_PhysMem_RAImportAlloc,
	                               _MMU_PhysMem_RAImportFree,
	                               psCtx, /* priv */
	                               IMG_FALSE);
	if (psCtx->psPhysMemRA == NULL)
	{
		OSFreeMem(psCtx->pszPhysMemRAName);
		psCtx->pszPhysMemRAName = NULL;
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e3;
	}

	/* Setup cleanup meta data to check if a MMU context
	 * has been destroyed and should not be accessed anymore */
	psCtx->psCleanupData = OSAllocMem(sizeof(*(psCtx->psCleanupData)));
	if (psCtx->psCleanupData == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Call to OSAllocMem failed", __func__));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e4;
	}

	OSLockCreate(&psCtx->psCleanupData->hCleanupLock);
	psCtx->psCleanupData->bMMUContextExists = IMG_TRUE;
	dllist_init(&psCtx->psCleanupData->sMMUCtxCleanupItemsHead);
	OSAtomicWrite(&psCtx->psCleanupData->iRef, 1);

	/* allocate the base level object */
	/*
	   Note: Although this is not required by the this file until
	         the 1st allocation is made, a device specific callback
	         might request the base object address so we allocate
	         it up front.
	 */
	if (_PxMemAlloc(psMMUContext,
	                ui32BaseObjects,
	                psConfig,
	                psDevAttrs->eTopLevel,
	                &psMMUContext->sBaseLevelInfo.sMemDesc,
	                psDevAttrs->ui32BaseAlign))
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Failed to alloc level 1 object", __func__));
		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto e5;
	}

	dllist_init(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead);

	psMMUContext->sBaseLevelInfo.ui32NumOfEntries = ui32BaseObjects;
	psMMUContext->sBaseLevelInfo.ui32RefCount = 0;

	eError = OSLockCreate(&psMMUContext->hLock);

	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Failed to create lock for MMU_CONTEXT", __func__));
		goto e6;
	}

	/* return context */
	*ppsMMUContext = psMMUContext;

	return PVRSRV_OK;

	e6:
	_PxMemFree(psMMUContext, &psMMUContext->sBaseLevelInfo.sMemDesc, psDevAttrs->eTopLevel);
	e5:
	OSFreeMem(psCtx->psCleanupData);
	e4:
	RA_Delete(psCtx->psPhysMemRA);
	e3:
	OSFreeMem(psCtx->pszPhysMemRAName);
	e2:
	OSFreeMem(psCtx);
	e1:
	OSFreeMem(psMMUContext);
	e0:
	return eError;
}

/*
	MMU_ContextDestroy
 */
void
MMU_ContextDestroy (MMU_CONTEXT *psMMUContext)
{
	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
	PDLLIST_NODE psNode, psNextNode;

	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *) psMMUContext->psDevNode;
	MMU_CTX_CLEANUP_DATA *psCleanupData = psMMUContext->psPhysMemCtx->psCleanupData;

	PVR_DPF((PVR_DBG_MESSAGE, "%s: Enter", __func__));

	if (psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK)
	{
		/* There should be no way to get here with live pages unless
		   there is a bug in this module or the MM code */
		PVR_ASSERT(psMMUContext->sBaseLevelInfo.ui32RefCount == 0);
	}

	/* Cleanup lock must be acquired before MMUContext lock. Reverse order
	 * may lead to a deadlock and is reported by lockdep. */
	OSLockAcquire(psCleanupData->hCleanupLock);
	OSLockAcquire(psMMUContext->hLock);

	/* Free the top level MMU object - will be put on defer free list.
	 * This has to be done before the step below that will empty the
	 * defer-free list. */
	_PxMemFree(psMMUContext,
	           &psMMUContext->sBaseLevelInfo.sMemDesc,
	           psMMUContext->psDevAttrs->eTopLevel);

	/* Empty the temporary defer-free list of Px */
	_FreeMMUMapping(psDevNode, &psMMUContext->psPhysMemCtx->sTmpMMUMappingHead);
	PVR_ASSERT(dllist_is_empty(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead));

	/* Empty the defer free list so the cleanup thread will
	 * not have to access any MMU context related structures anymore */
	dllist_foreach_node(&psCleanupData->sMMUCtxCleanupItemsHead,
	                    psNode,
	                    psNextNode)
	{
		MMU_CLEANUP_ITEM *psCleanup = IMG_CONTAINER_OF(psNode,
		                                               MMU_CLEANUP_ITEM,
		                                               sMMUCtxCleanupItem);

		_FreeMMUMapping(psDevNode, &psCleanup->sMMUMappingHead);

		dllist_remove_node(psNode);
	}
	PVR_ASSERT(dllist_is_empty(&psCleanupData->sMMUCtxCleanupItemsHead));

	psCleanupData->bMMUContextExists = IMG_FALSE;

	/* Free physmem context */
	RA_Delete(psMMUContext->psPhysMemCtx->psPhysMemRA);
	psMMUContext->psPhysMemCtx->psPhysMemRA = NULL;
	OSFreeMem(psMMUContext->psPhysMemCtx->pszPhysMemRAName);
	psMMUContext->psPhysMemCtx->pszPhysMemRAName = NULL;

	OSFreeMem(psMMUContext->psPhysMemCtx);

	OSLockRelease(psMMUContext->hLock);

	OSLockRelease(psCleanupData->hCleanupLock);

	if (OSAtomicDecrement(&psCleanupData->iRef) == 0)
	{
		OSLockDestroy(psCleanupData->hCleanupLock);
		OSFreeMem(psCleanupData);
	}

#if defined(SUPPORT_GPUVIRT_VALIDATION)
	RemovePidOSidCoupling(OSGetCurrentClientProcessIDKM());
#endif

	OSLockDestroy(psMMUContext->hLock);

	/* free the context itself. */
	OSFreeMem(psMMUContext);
	/*not nulling pointer, copy on stack*/

	PVR_DPF((PVR_DBG_MESSAGE, "%s: Exit", __func__));
}

/*
	MMU_Alloc
 */
PVRSRV_ERROR
MMU_Alloc (MMU_CONTEXT *psMMUContext,
           IMG_DEVMEM_SIZE_T uSize,
           IMG_DEVMEM_SIZE_T *puActualSize,
           IMG_UINT32 uiProtFlags,
           IMG_DEVMEM_SIZE_T uDevVAddrAlignment,
           IMG_DEV_VIRTADDR *psDevVAddr,
           IMG_UINT32 uiLog2PageSize)
{
	PVRSRV_ERROR eError;
	IMG_DEV_VIRTADDR sDevVAddrEnd;

	const MMU_PxE_CONFIG *psPDEConfig;
	const MMU_PxE_CONFIG *psPTEConfig;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;

	MMU_DEVICEATTRIBS *psDevAttrs;
	IMG_HANDLE hPriv;

#if !defined (DEBUG)
	PVR_UNREFERENCED_PARAMETER(uDevVAddrAlignment);
#endif
	PVR_DPF((PVR_DBG_MESSAGE,
			"%s: uSize=" IMG_DEVMEM_SIZE_FMTSPEC
			", uiProtFlags=0x%x, align="IMG_DEVMEM_ALIGN_FMTSPEC,
			__func__, uSize, uiProtFlags, uDevVAddrAlignment));

	/* check params */
	if (!psMMUContext || !psDevVAddr || !puActualSize)
	{
		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid parameter", __func__));
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	psDevAttrs = psMMUContext->psDevAttrs;

	eError = psDevAttrs->pfnGetPageSizeConfiguration(uiLog2PageSize,
	                                                 &psPDEConfig,
	                                                 &psPTEConfig,
	                                                 &psDevVAddrConfig,
	                                                 &hPriv);
	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Failed to get config info (%s)",
				__func__, PVRSRVGetErrorString(eError)));
		return eError;
	}

	/* size and alignment must be datapage granular */
	if (((psDevVAddr->uiAddr & psDevVAddrConfig->uiPageOffsetMask) != 0)
			|| ((uSize & psDevVAddrConfig->uiPageOffsetMask) != 0))
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: invalid address or size granularity",
				__func__));
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	sDevVAddrEnd = *psDevVAddr;
	sDevVAddrEnd.uiAddr += uSize;

	OSLockAcquire(psMMUContext->hLock);
	eError = _AllocPageTables(psMMUContext, *psDevVAddr, sDevVAddrEnd, uiLog2PageSize);
	OSLockRelease(psMMUContext->hLock);

	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: _AllocPageTables failed",
				__func__));
		return PVRSRV_ERROR_MMU_FAILED_TO_ALLOCATE_PAGETABLES;
	}

	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);

	return PVRSRV_OK;
}

/*
	MMU_Free
 */
void
MMU_Free (MMU_CONTEXT *psMMUContext,
          IMG_DEV_VIRTADDR sDevVAddr,
          IMG_DEVMEM_SIZE_T uiSize,
          IMG_UINT32 uiLog2DataPageSize)
{
	IMG_DEV_VIRTADDR sDevVAddrEnd;

	if (psMMUContext == NULL)
	{
		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid parameter", __func__));
		return;
	}

	PVR_DPF((PVR_DBG_MESSAGE, "%s: Freeing DevVAddr " IMG_DEV_VIRTADDR_FMTSPEC,
			__func__, sDevVAddr.uiAddr));

	/* ensure the address range to free is inside the heap */
	sDevVAddrEnd = sDevVAddr;
	sDevVAddrEnd.uiAddr += uiSize;

	/* The Cleanup lock has to be taken before the MMUContext hLock to
	 * prevent deadlock scenarios. It is necessary only for parts of
	 * _SetupCleanup_FreeMMUMapping though.*/
	OSLockAcquire(psMMUContext->psPhysMemCtx->psCleanupData->hCleanupLock);

	OSLockAcquire(psMMUContext->hLock);

	_FreePageTables(psMMUContext,
	                sDevVAddr,
	                sDevVAddrEnd,
	                uiLog2DataPageSize);

	_SetupCleanup_FreeMMUMapping(psMMUContext->psDevNode,
	                             psMMUContext->psPhysMemCtx);

	OSLockRelease(psMMUContext->hLock);

	OSLockRelease(psMMUContext->psPhysMemCtx->psCleanupData->hCleanupLock);

	return;

}

PVRSRV_ERROR
MMU_MapPages(MMU_CONTEXT *psMMUContext,
             PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
             IMG_DEV_VIRTADDR sDevVAddrBase,
             PMR *psPMR,
             IMG_UINT32 ui32PhysPgOffset,
             IMG_UINT32 ui32MapPageCount,
             IMG_UINT32 *paui32MapIndices,
             IMG_UINT32 uiLog2HeapPageSize)
{
	PVRSRV_ERROR eError;
	IMG_HANDLE hPriv;

	MMU_Levelx_INFO *psLevel = NULL;

	MMU_Levelx_INFO *psPrevLevel = NULL;

	IMG_UINT32 uiPTEIndex = 0;
	IMG_UINT32 uiPageSize = (1 << uiLog2HeapPageSize);
	IMG_UINT32 uiLoop = 0;
	IMG_UINT32 ui32MappedCount = 0;
	IMG_DEVMEM_OFFSET_T uiPgOffset = 0;
	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;

	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0, uiDefProtFlags=0;
	MMU_PROTFLAGS_T uiMMUProtFlags = 0;

	const MMU_PxE_CONFIG *psConfig;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;

	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;

	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
	IMG_DEV_PHYADDR *psDevPAddr;
	IMG_DEV_PHYADDR sDevPAddr;
	IMG_BOOL *pbValid;
	IMG_BOOL bValid;
	IMG_BOOL bDummyBacking = IMG_FALSE, bZeroBacking = IMG_FALSE;
	IMG_BOOL bNeedBacking = IMG_FALSE;

#if defined(PDUMP)
	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
	IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset;

	PDUMPCOMMENT("Wire up Page Table entries to point to the Data Pages (%"IMG_INT64_FMTSPECd" bytes)",
	             (IMG_UINT64)(ui32MapPageCount * uiPageSize));
#endif /*PDUMP*/

#if defined(TC_MEMORY_CONFIG) || defined(PLATO_MEMORY_CONFIG)
	/* We're aware that on TC based platforms, accesses from GPU to CPU_LOCAL
	 * allocated DevMem fail, so we forbid mapping such a PMR into device mmu */
	if (PMR_Flags(psPMR) & PVRSRV_MEMALLOCFLAG_CPU_LOCAL)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Mapping a CPU_LOCAL PMR to device is forbidden on this platform", __func__));
		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
	}
#endif

	/* Validate the most essential parameters */
	if ((NULL == psMMUContext) || (0 == sDevVAddrBase.uiAddr) || (NULL == psPMR))
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Invalid mapping parameter issued",
				__func__));
		eError = PVRSRV_ERROR_INVALID_PARAMS;
		goto e0;
	}

	/* Allocate memory for page-frame-numbers and validity states,
	   N.B. assert could be triggered by an illegal uiSizeBytes */
	if (ui32MapPageCount > PMR_MAX_TRANSLATION_STACK_ALLOC)
	{
		psDevPAddr = OSAllocMem(ui32MapPageCount * sizeof(IMG_DEV_PHYADDR));
		if (psDevPAddr == NULL)
		{
			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate PMR device PFN list"));
			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
			goto e0;
		}

		pbValid = OSAllocMem(ui32MapPageCount * sizeof(IMG_BOOL));
		if (pbValid == NULL)
		{
			/* Should allocation fail, clean-up here before exit */
			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate PMR device PFN state"));
			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
			OSFreeMem(psDevPAddr);
			goto e0;
		}
	}
	else
	{
		psDevPAddr = asDevPAddr;
		pbValid	= abValid;
	}

	/* Get the Device physical addresses of the pages we are trying to map
	 * In the case of non indexed mapping we can get all addresses at once */
	if (NULL == paui32MapIndices)
	{
		eError = PMR_DevPhysAddr(psPMR,
		                         uiLog2HeapPageSize,
		                         ui32MapPageCount,
		                         (ui32PhysPgOffset << uiLog2HeapPageSize),
		                         psDevPAddr,
		                         pbValid);
		if (eError != PVRSRV_OK)
		{
			goto e1;
		}
	}

	/*Get the Page table level configuration */
	_MMU_GetPTConfig(psMMUContext,
	                 (IMG_UINT32) uiLog2HeapPageSize,
	                 &psConfig,
	                 &hPriv,
	                 &psDevVAddrConfig);

	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
	                                 uiMappingFlags,
	                                 &uiMMUProtFlags,
	                                 psMMUContext);
	if (eError != PVRSRV_OK)
	{
		goto e2;
	}

	/* Callback to get device specific protection flags */
	if (psConfig->uiBytesPerEntry == 8)
	{
		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE),
		                                                                  uiLog2HeapPageSize);
	}
	else if (psConfig->uiBytesPerEntry == 4)
	{
		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE));
	}
	else
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: The page table entry byte length is not supported",
				__func__));
		eError = PVRSRV_ERROR_INVALID_PARAMS;
		goto e2;
	}

	if (PMR_IsSparse(psPMR))
	{
		/* We know there will not be 4G number of PMR's */
		bDummyBacking = PVRSRV_IS_SPARSE_DUMMY_BACKING_REQUIRED(PMR_Flags(psPMR));
		if (bDummyBacking)
		{
			bZeroBacking = PVRSRV_IS_SPARSE_ZERO_BACKING_REQUIRED(PMR_Flags(psPMR));
		}
	}

	OSLockAcquire(psMMUContext->hLock);

	for (uiLoop = 0; uiLoop < ui32MapPageCount; uiLoop++)
	{

#if defined(PDUMP)
		IMG_DEVMEM_OFFSET_T uiNextSymName;
#endif /*PDUMP*/

		if (NULL != paui32MapIndices)
		{
			uiPgOffset = paui32MapIndices[uiLoop];

			/*Calculate the Device Virtual Address of the page */
			sDevVAddr.uiAddr = sDevVAddrBase.uiAddr + (uiPgOffset * uiPageSize);
			/* Get the physical address to map */
			eError = PMR_DevPhysAddr(psPMR,
			                         uiLog2HeapPageSize,
			                         1,
			                         uiPgOffset * uiPageSize,
			                         &sDevPAddr,
			                         &bValid);
			if (eError != PVRSRV_OK)
			{
				goto e3;
			}
		}
		else
		{
			uiPgOffset = uiLoop + ui32PhysPgOffset;
			sDevPAddr = psDevPAddr[uiLoop];
			bValid = pbValid[uiLoop];
		}

		uiDefProtFlags = uiProtFlags;
		/*
			The default value of the entry is invalid so we don't need to mark
			it as such if the page wasn't valid, we just advance pass that address
		 */
		if (bValid || bDummyBacking)
		{

			if (!bValid)
			{
				if (bZeroBacking)
				{
					sDevPAddr.uiAddr = psMMUContext->psDevNode->sDevZeroPage.ui64PgPhysAddr;
					/* Ensure the zero back page PTE is read only */
					uiDefProtFlags = uiProtFlagsReadOnly;
				}
				else
				{
					sDevPAddr.uiAddr = psMMUContext->psDevNode->sDummyPage.ui64PgPhysAddr;
				}
			}
			else
			{
				/* check the physical alignment of the memory to map */
				PVR_ASSERT((sDevPAddr.uiAddr & (uiPageSize-1)) == 0);
			}

#if defined(DEBUG)
			{
				IMG_INT32	i32FeatureVal = 0;
				IMG_UINT32 ui32BitLength = FloorLog2(sDevPAddr.uiAddr);

				i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psMMUContext->psDevNode, PHYS_BUS_WIDTH);
				do {
					/* i32FeatureVal can be negative for cases where this feature is undefined
					 * In that situation we need to bail out than go ahead with debug comparison */
					if (0 > i32FeatureVal)
						break;

					if (ui32BitLength > i32FeatureVal)
					{
						PVR_DPF((PVR_DBG_ERROR,
								"%s Failed. The physical address bitlength (%d)"
								" is greater than the chip can handle (%d).",
								__func__, ui32BitLength, i32FeatureVal));

						PVR_ASSERT(ui32BitLength <= i32FeatureVal);
						eError = PVRSRV_ERROR_INVALID_PARAMS;
						goto e3;
					}
				}while(0);
			}
#endif /*DEBUG*/

#if defined(PDUMP)
			if (bValid)
			{
				eError = PMR_PDumpSymbolicAddr(psPMR, uiPgOffset * uiPageSize,
				                               sizeof(aszMemspaceName), &aszMemspaceName[0],
				                               sizeof(aszSymbolicAddress), &aszSymbolicAddress[0],
				                               &uiSymbolicAddrOffset,
				                               &uiNextSymName);
				PVR_ASSERT(eError == PVRSRV_OK);
			}
#endif /*PDUMP*/

			psPrevLevel = psLevel;
			/* Calculate PT index and get new table descriptor */
			_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
			               &psLevel, &uiPTEIndex);

			if (psPrevLevel == psLevel)
			{
				/*
				 * Sparse allocations may have page offsets which
				 * decrement as well as increment, so make sure we
				 * update the range we will flush correctly.
				 */
				if (uiPTEIndex > uiFlushEnd)
					uiFlushEnd = uiPTEIndex;
				else if (uiPTEIndex < uiFlushStart)
					uiFlushStart = uiPTEIndex;
			}
			else
			{
				/* Flush if we moved to another psLevel, i.e. page table */
				if (psPrevLevel != NULL)
				{
					eError = psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
					                                                &psPrevLevel->sMemDesc.psMapping->sMemHandle,
					                                                uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
					                                                (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
					if (eError != PVRSRV_OK)
						goto e3;
				}

				uiFlushStart = uiPTEIndex;
				uiFlushEnd = uiFlushStart;
			}

			HTBLOGK(HTB_SF_MMU_PAGE_OP_MAP,
			        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
			        HTBLOG_U64_BITS_HIGH(sDevPAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevPAddr.uiAddr));

			eError = _SetupPTE(psMMUContext,
			                   psLevel,
			                   uiPTEIndex,
			                   psConfig,
			                   &sDevPAddr,
			                   IMG_FALSE,
#if defined(PDUMP)
			                   (bValid)?aszMemspaceName:(psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName),
			                		   ((bValid)?aszSymbolicAddress:((bZeroBacking)?DEV_ZERO_PAGE:DUMMY_PAGE)),
			                		   (bValid)?uiSymbolicAddrOffset:0,
#endif /*PDUMP*/
			                    uiDefProtFlags);


			if (eError != PVRSRV_OK)
			{
				PVR_DPF((PVR_DBG_ERROR, "%s: Mapping failed", __func__));
				goto e3;
			}

			if (bValid)
			{
				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
				PVR_DPF ((PVR_DBG_MESSAGE,
						"%s: devVAddr=" IMG_DEV_VIRTADDR_FMTSPEC ", "
						"size=" IMG_DEVMEM_OFFSET_FMTSPEC,
						__func__,
						sDevVAddr.uiAddr,
						uiPgOffset * uiPageSize));

				ui32MappedCount++;
			}
		}

		sDevVAddr.uiAddr += uiPageSize;
	}

	/* Flush the last level we touched */
	if (psLevel != NULL)
	{
		eError = psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
		                                                &psLevel->sMemDesc.psMapping->sMemHandle,
		                                                uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
		                                                (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
		if (eError != PVRSRV_OK)
			goto e3;
	}

	OSLockRelease(psMMUContext->hLock);

	_MMU_PutPTConfig(psMMUContext, hPriv);

	if (psDevPAddr != asDevPAddr)
	{
		OSFreeMem(pbValid);
		OSFreeMem(psDevPAddr);
	}

	/* Flush TLB for PTs*/
	psMMUContext->psDevNode->pfnMMUCacheInvalidate(psMMUContext->psDevNode,
	                                               psMMUContext->hDevData,
	                                               MMU_LEVEL_1,
	                                               IMG_FALSE);

#if defined(PDUMP)
	PDUMPCOMMENT("Wired up %d Page Table entries (out of %d)", ui32MappedCount, ui32MapPageCount);
#endif /*PDUMP*/

	return PVRSRV_OK;

	e3:
	OSLockRelease(psMMUContext->hLock);

	if (PMR_IsSparse(psPMR) && PVRSRV_IS_SPARSE_DUMMY_BACKING_REQUIRED(uiMappingFlags))
	{
		bNeedBacking = IMG_TRUE;
	}

	MMU_UnmapPages(psMMUContext,(bNeedBacking)?uiMappingFlags:0, sDevVAddrBase, uiLoop, paui32MapIndices, uiLog2HeapPageSize, PMR_IsSparse(psPMR));
	e2:
	_MMU_PutPTConfig(psMMUContext, hPriv);
	e1:
	if (psDevPAddr != asDevPAddr)
	{
		OSFreeMem(pbValid);
		OSFreeMem(psDevPAddr);
	}
	e0:
	return eError;
}

/*
	MMU_UnmapPages
 */
void
MMU_UnmapPages (MMU_CONTEXT *psMMUContext,
                PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
                IMG_DEV_VIRTADDR sDevVAddrBase,
                IMG_UINT32 ui32PageCount,
                IMG_UINT32 *pai32FreeIndices,
                IMG_UINT32 uiLog2PageSize,
                PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags)
{
	IMG_UINT32 uiPTEIndex = 0, ui32Loop=0;
	IMG_UINT32 uiPageSize = 1 << uiLog2PageSize;
	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
	MMU_Levelx_INFO *psLevel = NULL;
	MMU_Levelx_INFO *psPrevLevel = NULL;
	IMG_HANDLE hPriv;
	const MMU_PxE_CONFIG *psConfig;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0;
	MMU_PROTFLAGS_T uiMMUProtFlags = 0, uiMMUReadOnlyProtFlags = 0;
	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
	IMG_DEV_PHYADDR sBackingPgDevPhysAddr;
	IMG_BOOL bUnmap = IMG_TRUE, bDummyBacking = IMG_FALSE, bZeroBacking = IMG_FALSE;
	IMG_CHAR *pcBackingPageName = NULL;

#if defined(PDUMP)
	PDUMPCOMMENT("Invalidate %d entries in page tables for virtual range: 0x%010"IMG_UINT64_FMTSPECX" to 0x%010"IMG_UINT64_FMTSPECX,
	             ui32PageCount,
	             (IMG_UINT64)sDevVAddr.uiAddr,
	             ((IMG_UINT64)sDevVAddr.uiAddr) + (uiPageSize*ui32PageCount)-1);
#endif
	bDummyBacking = PVRSRV_IS_SPARSE_DUMMY_BACKING_REQUIRED(uiMemAllocFlags);
	bZeroBacking = PVRSRV_IS_SPARSE_ZERO_BACKING_REQUIRED(uiMemAllocFlags);

	if (bZeroBacking)
	{
		sBackingPgDevPhysAddr.uiAddr = psMMUContext->psDevNode->sDevZeroPage.ui64PgPhysAddr;
		pcBackingPageName = DEV_ZERO_PAGE;
	}
	else
	{
		sBackingPgDevPhysAddr.uiAddr = psMMUContext->psDevNode->sDummyPage.ui64PgPhysAddr;
		pcBackingPageName = DUMMY_PAGE;
	}

	bUnmap = (uiMappingFlags)? !bDummyBacking : IMG_TRUE;
	/* Get PT and address configs */
	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2PageSize,
	                 &psConfig, &hPriv, &psDevVAddrConfig);

	if (_MMU_ConvertDevMemFlags(bUnmap,
	                            uiMappingFlags,
	                            &uiMMUProtFlags,
	                            psMMUContext) != PVRSRV_OK)
	{
		return;
	}

	uiMMUReadOnlyProtFlags = (uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE) | MMU_PROTFLAGS_READABLE;

	/* Callback to get device specific protection flags */
	if (psConfig->uiBytesPerEntry == 4)
	{
		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUReadOnlyProtFlags);
	}
	else if (psConfig->uiBytesPerEntry == 8)
	{
		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2PageSize);
		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUReadOnlyProtFlags, uiLog2PageSize);
	}


	OSLockAcquire(psMMUContext->hLock);

	/* Unmap page by page */
	while (ui32Loop < ui32PageCount)
	{
		if (NULL != pai32FreeIndices)
		{
			/*Calculate the Device Virtual Address of the page */
			sDevVAddr.uiAddr = sDevVAddrBase.uiAddr +
					pai32FreeIndices[ui32Loop] * (IMG_UINT64) uiPageSize;
		}

		psPrevLevel = psLevel;
		/* Calculate PT index and get new table descriptor */
		_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
		               &psLevel, &uiPTEIndex);

		if (psPrevLevel == psLevel)
		{
			/*
			 * Sparse allocations may have page offsets which
			 * decrement as well as increment, so make sure we
			 * update the range we will flush correctly.
			 */
			if (uiPTEIndex > uiFlushEnd)
				uiFlushEnd = uiPTEIndex;
			else if (uiPTEIndex < uiFlushStart)
				uiFlushStart = uiPTEIndex;
		}
		else
		{
			/* Flush if we moved to another psLevel, i.e. page table */
			if (psPrevLevel != NULL)
			{
				psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
				                                       &psPrevLevel->sMemDesc.psMapping->sMemHandle,
				                                       uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
				                                       (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
			}

			uiFlushStart = uiPTEIndex;
			uiFlushEnd = uiFlushStart;
		}

		HTBLOGK(HTB_SF_MMU_PAGE_OP_UNMAP,
		        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr));

		if (_SetupPTE(psMMUContext,
		              psLevel,
		              uiPTEIndex,
		              psConfig,
		              (bDummyBacking)? &sBackingPgDevPhysAddr : &gsBadDevPhyAddr,
		            		  bUnmap,
#if defined(PDUMP)
		            		  (bDummyBacking)? (psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName): NULL,
		            		  (bDummyBacking)? pcBackingPageName: NULL,
		            		  0U,
#endif
		            		  (bZeroBacking)? uiProtFlagsReadOnly: uiProtFlags) != PVRSRV_OK)
		{
			goto e0;
		}

		/* Check we haven't wrapped around */
		PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
		ui32Loop++;
		sDevVAddr.uiAddr += uiPageSize;
	}

	/* Flush the last level we touched */
	if (psLevel != NULL)
	{
		psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
		                                       &psLevel->sMemDesc.psMapping->sMemHandle,
		                                       uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
		                                       (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
	}

	OSLockRelease(psMMUContext->hLock);

	_MMU_PutPTConfig(psMMUContext, hPriv);

	/* Flush TLB for PTs*/
	psMMUContext->psDevNode->pfnMMUCacheInvalidate(psMMUContext->psDevNode,
	                                               psMMUContext->hDevData,
	                                               MMU_LEVEL_1,
	                                               IMG_TRUE);

	return;

	e0:
	_MMU_PutPTConfig(psMMUContext, hPriv);
	PVR_DPF((PVR_DBG_ERROR, "MMU_UnmapPages: Failed to map/unmap page table"));
	PVR_ASSERT(0);
	OSLockRelease(psMMUContext->hLock);
	return;
}

PVRSRV_ERROR
MMU_MapPMRFast (MMU_CONTEXT *psMMUContext,
                IMG_DEV_VIRTADDR sDevVAddrBase,
                const PMR *psPMR,
                IMG_DEVMEM_SIZE_T uiSizeBytes,
                PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
                IMG_UINT32 uiLog2HeapPageSize)
{
	PVRSRV_ERROR eError = PVRSRV_OK;
	IMG_UINT32 uiCount, i;
	IMG_UINT32 uiPageSize = 1 << uiLog2HeapPageSize;
	IMG_UINT32 uiPTEIndex = 0;
	IMG_UINT64 uiProtFlags;
	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
	MMU_Levelx_INFO *psLevel = NULL;
	IMG_HANDLE hPriv;
	const MMU_PxE_CONFIG *psConfig;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
	IMG_DEV_PHYADDR *psDevPAddr;
	IMG_BOOL *pbValid;
	IMG_UINT32 uiFlushStart = 0;

#if defined(PDUMP)
	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
	IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset;
	IMG_UINT32 ui32MappedCount = 0;
	PDUMPCOMMENT("Wire up Page Table entries to point to the Data Pages (%"IMG_INT64_FMTSPECd" bytes)", uiSizeBytes);
#endif /*PDUMP*/

	/* We should verify the size and contiguity when supporting variable page size */

	PVR_ASSERT (psMMUContext != NULL);
	PVR_ASSERT (psPMR != NULL);

#if defined(TC_MEMORY_CONFIG) || defined(PLATO_MEMORY_CONFIG)
	/* We're aware that on TC based platforms, accesses from GPU to CPU_LOCAL
	 * allocated DevMem fail, so we forbid mapping such a PMR into device mmu */
	if (PMR_Flags(psPMR) & PVRSRV_MEMALLOCFLAG_CPU_LOCAL)
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: Mapping a CPU_LOCAL PMR to device is forbidden on this platform", __func__));
		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
	}
#endif

	/* Allocate memory for page-frame-numbers and validity states,
	   N.B. assert could be triggered by an illegal uiSizeBytes */
	uiCount = uiSizeBytes >> uiLog2HeapPageSize;
	PVR_ASSERT((IMG_DEVMEM_OFFSET_T)uiCount << uiLog2HeapPageSize == uiSizeBytes);
	if (uiCount > PMR_MAX_TRANSLATION_STACK_ALLOC)
	{
		psDevPAddr = OSAllocMem(uiCount * sizeof(IMG_DEV_PHYADDR));
		if (psDevPAddr == NULL)
		{
			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate PMR device PFN list"));
			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
			goto return_error;
		}

		pbValid = OSAllocMem(uiCount * sizeof(IMG_BOOL));
		if (pbValid == NULL)
		{
			/* Should allocation fail, clean-up here before exit */
			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate PMR device PFN state"));
			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
			OSFreeMem(psDevPAddr);
			goto free_paddr_array;
		}
	}
	else
	{
		psDevPAddr = asDevPAddr;
		pbValid	= abValid;
	}

	/* Get general PT and address configs */
	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2HeapPageSize,
	                 &psConfig, &hPriv, &psDevVAddrConfig);

	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
	                                 uiMappingFlags,
	                                 &uiMMUProtFlags,
	                                 psMMUContext);
	if (eError != PVRSRV_OK)
	{
		goto put_mmu_context;
	}

	/* Callback to get device specific protection flags */

	if (psConfig->uiBytesPerEntry == 8)
	{
		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
	}
	else if (psConfig->uiBytesPerEntry == 4)
	{
		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
	}
	else
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: The page table entry byte length is not supported",
				__func__));
		eError = PVRSRV_ERROR_MMU_CONFIG_IS_WRONG;
		goto put_mmu_context;
	}


	/* "uiSize" is the amount of contiguity in the underlying
	   page.  Normally this would be constant for the system, but,
	   that constant needs to be communicated, in case it's ever
	   different; caller guarantees that PMRLockSysPhysAddr() has
	   already been called */
	eError = PMR_DevPhysAddr(psPMR,
	                         uiLog2HeapPageSize,
	                         uiCount,
	                         0,
	                         psDevPAddr,
	                         pbValid);
	if (eError != PVRSRV_OK)
	{
		goto put_mmu_context;
	}

	OSLockAcquire(psMMUContext->hLock);

	_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
	               &psLevel, &uiPTEIndex);
	uiFlushStart = uiPTEIndex;

	/* Map in all pages of that PMR page by page*/
	for (i=0, uiCount=0; uiCount < uiSizeBytes; i++)
	{
#if defined(DEBUG)
		{
			IMG_INT32	i32FeatureVal = 0;
			IMG_UINT32 ui32BitLength = FloorLog2(psDevPAddr[i].uiAddr);
			i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psMMUContext->psDevNode, PHYS_BUS_WIDTH);
			do {
				if (0 > i32FeatureVal)
					break;

				if (ui32BitLength > i32FeatureVal)
				{
					PVR_DPF((PVR_DBG_ERROR,
							"%s Failed. The physical address bitlength (%d)"
							" is greater than the chip can handle (%d).",
							__func__, ui32BitLength, i32FeatureVal));

					PVR_ASSERT(ui32BitLength <= i32FeatureVal);
					eError = PVRSRV_ERROR_INVALID_PARAMS;
					OSLockRelease(psMMUContext->hLock);
					goto put_mmu_context;
				}
			}while(0);
		}
#endif /*DEBUG*/
#if defined(PDUMP)
		{
			IMG_DEVMEM_OFFSET_T uiNextSymName;

			eError = PMR_PDumpSymbolicAddr(psPMR, uiCount,
			                               sizeof(aszMemspaceName), &aszMemspaceName[0],
			                               sizeof(aszSymbolicAddress), &aszSymbolicAddress[0],
			                               &uiSymbolicAddrOffset,
			                               &uiNextSymName);
			PVR_ASSERT(eError == PVRSRV_OK);
			ui32MappedCount++;
		}
#endif /*PDUMP*/

		HTBLOGK(HTB_SF_MMU_PAGE_OP_PMRMAP,
		        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
		        HTBLOG_U64_BITS_HIGH(psDevPAddr[i].uiAddr), HTBLOG_U64_BITS_LOW(psDevPAddr[i].uiAddr));

		/* Set the PT entry with the specified address and protection flags */
		eError = _SetupPTE(psMMUContext, psLevel, uiPTEIndex,
		                   psConfig, &psDevPAddr[i], IMG_FALSE,
#if defined(PDUMP)
		                   aszMemspaceName,
		                   aszSymbolicAddress,
		                   uiSymbolicAddrOffset,
#endif /*PDUMP*/
		                   uiProtFlags);
		if (eError != PVRSRV_OK)
			goto unlock_mmu_context;

		sDevVAddr.uiAddr += uiPageSize;
		uiCount += uiPageSize;

		/* Calculate PT index and get new table descriptor */
		if (uiPTEIndex < (psDevVAddrConfig->uiNumEntriesPT - 1) && (uiCount != uiSizeBytes))
		{
			uiPTEIndex++;
		}
		else
		{
			eError = psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
			                                                &psLevel->sMemDesc.psMapping->sMemHandle,
			                                                uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
			                                                (uiPTEIndex+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
			if (eError != PVRSRV_OK)
				goto unlock_mmu_context;


			_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
			               &psLevel, &uiPTEIndex);
			uiFlushStart = uiPTEIndex;
		}
	}

	OSLockRelease(psMMUContext->hLock);


	_MMU_PutPTConfig(psMMUContext, hPriv);

	if (psDevPAddr != asDevPAddr)
	{
		OSFreeMem(pbValid);
		OSFreeMem(psDevPAddr);
	}

	/* Flush TLB for PTs*/
	psMMUContext->psDevNode->pfnMMUCacheInvalidate(psMMUContext->psDevNode,
	                                               psMMUContext->hDevData,
	                                               MMU_LEVEL_1,
	                                               IMG_FALSE);

#if defined(PDUMP)
	PDUMPCOMMENT("Wired up %d Page Table entries (out of %d)", ui32MappedCount, i);
#endif /*PDUMP*/

	return PVRSRV_OK;

unlock_mmu_context:
	OSLockRelease(psMMUContext->hLock);
	MMU_UnmapPMRFast(psMMUContext,
	                 sDevVAddrBase,
	                 uiSizeBytes >> uiLog2HeapPageSize,
	                 uiLog2HeapPageSize);
put_mmu_context:
	_MMU_PutPTConfig(psMMUContext, hPriv);

	if (pbValid != abValid)
	{
		OSFreeMem(pbValid);
	}

free_paddr_array:
	if (psDevPAddr != asDevPAddr)
	{
		OSFreeMem(psDevPAddr);
	}

return_error:
	PVR_ASSERT(eError == PVRSRV_OK);
	return eError;
}

/*
    MMU_UnmapPages
 */
void
MMU_UnmapPMRFast(MMU_CONTEXT *psMMUContext,
                 IMG_DEV_VIRTADDR sDevVAddrBase,
                 IMG_UINT32 ui32PageCount,
                 IMG_UINT32 uiLog2PageSize)
{
	IMG_UINT32 uiPTEIndex = 0, ui32Loop=0;
	IMG_UINT32 uiPageSize = 1 << uiLog2PageSize;
	MMU_Levelx_INFO *psLevel = NULL;
	IMG_HANDLE hPriv;
	const MMU_PxE_CONFIG *psConfig;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
	IMG_UINT64 uiProtFlags = 0;
	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
	IMG_UINT64 uiEntry = 0;
	IMG_UINT32 uiFlushStart = 0;

#if defined(PDUMP)
	PDUMPCOMMENT("Invalidate %d entries in page tables for virtual range: 0x%010"IMG_UINT64_FMTSPECX" to 0x%010"IMG_UINT64_FMTSPECX,
	             ui32PageCount,
	             (IMG_UINT64)sDevVAddr.uiAddr,
	             ((IMG_UINT64)sDevVAddr.uiAddr) + (uiPageSize*ui32PageCount)-1);
#endif

	/* Get PT and address configs */
	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2PageSize,
	                 &psConfig, &hPriv, &psDevVAddrConfig);

	if (_MMU_ConvertDevMemFlags(IMG_TRUE,
	                            0,
	                            &uiMMUProtFlags,
	                            psMMUContext) != PVRSRV_OK)
	{
		return;
	}

	/* Callback to get device specific protection flags */

	if (psConfig->uiBytesPerEntry == 8)
	{
		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2PageSize);

		/* Fill the entry with a bad address but leave space for protection flags */
		uiEntry = (gsBadDevPhyAddr.uiAddr & ~psConfig->uiProtMask) | uiProtFlags;
	}
	else if (psConfig->uiBytesPerEntry == 4)
	{
		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);

		/* Fill the entry with a bad address but leave space for protection flags */
		uiEntry = (((IMG_UINT32) gsBadDevPhyAddr.uiAddr) & ~psConfig->uiProtMask) | (IMG_UINT32) uiProtFlags;
	}
	else
	{
		PVR_DPF((PVR_DBG_ERROR,
				"%s: The page table entry byte length is not supported",
				__func__));
		goto e0;
	}

	OSLockAcquire(psMMUContext->hLock);

	_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
	               &psLevel, &uiPTEIndex);
	uiFlushStart = uiPTEIndex;

	/* Unmap page by page and keep the loop as quick as possible.
	 * Only use parts of _SetupPTE that need to be executed. */
	while (ui32Loop < ui32PageCount)
	{

		/* Set the PT entry to invalid and poison it with a bad address */
		if (psConfig->uiBytesPerEntry == 8)
		{
			((IMG_UINT64*) psLevel->sMemDesc.pvCpuVAddr)[uiPTEIndex] = uiEntry;
		}
		else if (psConfig->uiBytesPerEntry == 4)
		{
			((IMG_UINT32*) psLevel->sMemDesc.pvCpuVAddr)[uiPTEIndex] = (IMG_UINT32) uiEntry;
		}
		else
		{
			PVR_DPF((PVR_DBG_ERROR,
					"%s: The page table entry byte length is not supported",
					__func__));
			goto e1;
		}

		/* Log modifications */
		HTBLOGK(HTB_SF_MMU_PAGE_OP_UNMAP,
		        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr));

		HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
		        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
		        uiPTEIndex, MMU_LEVEL_1,
		        HTBLOG_U64_BITS_HIGH(uiEntry), HTBLOG_U64_BITS_LOW(uiEntry),
		        IMG_FALSE);

#if defined (PDUMP)
		PDumpMMUDumpPxEntries(MMU_LEVEL_1,
		                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
		                      psLevel->sMemDesc.pvCpuVAddr,
		                      psLevel->sMemDesc.sDevPAddr,
		                      uiPTEIndex,
		                      1,
		                      NULL,
		                      NULL,
		                      0,
		                      psConfig->uiBytesPerEntry,
		                      psConfig->uiAddrLog2Align,
		                      psConfig->uiAddrShift,
		                      psConfig->uiAddrMask,
		                      psConfig->uiProtMask,
		                      psConfig->uiValidEnMask,
		                      0,
		                      psMMUContext->psDevAttrs->eMMUType);
#endif /*PDUMP*/

		sDevVAddr.uiAddr += uiPageSize;
		ui32Loop++;

		/* Calculate PT index and get new table descriptor */
		if (uiPTEIndex < (psDevVAddrConfig->uiNumEntriesPT - 1) && (ui32Loop != ui32PageCount))
		{
			uiPTEIndex++;
		}
		else
		{
			psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
			                                       &psLevel->sMemDesc.psMapping->sMemHandle,
			                                       uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
			                                       (uiPTEIndex+1 - uiFlushStart) * psConfig->uiBytesPerEntry);

			_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
			               &psLevel, &uiPTEIndex);
			uiFlushStart = uiPTEIndex;
		}
	}

	OSLockRelease(psMMUContext->hLock);

	_MMU_PutPTConfig(psMMUContext, hPriv);

	/* Flush TLB for PTs*/
	psMMUContext->psDevNode->pfnMMUCacheInvalidate(psMMUContext->psDevNode,
	                                               psMMUContext->hDevData,
	                                               MMU_LEVEL_1,
	                                               IMG_TRUE);

	return;

	e1:
	OSLockRelease(psMMUContext->hLock);
	_MMU_PutPTConfig(psMMUContext, hPriv);
	e0:
	PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map/unmap page table", __func__));
	PVR_ASSERT(0);
	return;
}

/*
	MMU_ChangeValidity
 */
PVRSRV_ERROR
MMU_ChangeValidity(MMU_CONTEXT *psMMUContext,
                   IMG_DEV_VIRTADDR sDevVAddr,
                   IMG_DEVMEM_SIZE_T uiNumPages,
                   IMG_UINT32 uiLog2PageSize,
                   IMG_BOOL bMakeValid,
                   PMR *psPMR)
{
	PVRSRV_ERROR eError = PVRSRV_OK;

	IMG_HANDLE hPriv;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
	const MMU_PxE_CONFIG *psConfig;
	MMU_Levelx_INFO *psLevel = NULL;
	IMG_UINT32 uiFlushStart = 0;
	IMG_UINT32 uiPTIndex = 0;
	IMG_UINT32 i;
	IMG_UINT32 uiPageSize = 1 << uiLog2PageSize;
	IMG_BOOL bValid;

#if defined(PDUMP)
	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
	IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset;
	IMG_DEVMEM_OFFSET_T uiNextSymName;

	PDUMPCOMMENT("Change valid bit of the data pages to %d (0x%"IMG_UINT64_FMTSPECX" - 0x%"IMG_UINT64_FMTSPECX")",
	             bMakeValid,
	             sDevVAddr.uiAddr,
	             sDevVAddr.uiAddr + (uiNumPages<<uiLog2PageSize) - 1);
#endif /*PDUMP*/

	/* We should verify the size and contiguity when supporting variable page size */
	PVR_ASSERT (psMMUContext != NULL);
	PVR_ASSERT (psPMR != NULL);

	/* Get general PT and address configs */
	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2PageSize,
	                 &psConfig, &hPriv, &psDevVAddrConfig);

	_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
	               &psLevel, &uiPTIndex);
	uiFlushStart = uiPTIndex;

	/* Do a page table walk and change attribute for every page in range. */
	for (i=0; i < uiNumPages;)
	{
		/* Set the entry */
		if (bMakeValid)
		{
			/* Only set valid if physical address exists (sparse allocs might have none)*/
			eError = PMR_IsOffsetValid(psPMR, uiLog2PageSize, 1, (IMG_DEVMEM_OFFSET_T) i << uiLog2PageSize, &bValid);
			if (eError != PVRSRV_OK)
			{
				PVR_DPF((PVR_DBG_ERROR, "Cannot determine validity of page table entries page"));
				goto e_exit;
			}

			if (bValid)
			{
				if (psConfig->uiBytesPerEntry == 8)
				{
					((IMG_UINT64 *) psLevel->sMemDesc.pvCpuVAddr)[uiPTIndex] |= (psConfig->uiValidEnMask);
				}
				else if (psConfig->uiBytesPerEntry == 4)
				{
					((IMG_UINT32 *) psLevel->sMemDesc.pvCpuVAddr)[uiPTIndex] |= (psConfig->uiValidEnMask);
				}
				else
				{
					eError = PVRSRV_ERROR_MMU_CONFIG_IS_WRONG;
					PVR_DPF((PVR_DBG_ERROR, "Cannot change page table entries due to wrong configuration"));
					goto e_exit;
				}
			}
		}
		else
		{
			if (psConfig->uiBytesPerEntry == 8)
			{
				((IMG_UINT64 *) psLevel->sMemDesc.pvCpuVAddr)[uiPTIndex] &= ~(psConfig->uiValidEnMask);
			}
			else if (psConfig->uiBytesPerEntry == 4)
			{
				((IMG_UINT32 *) psLevel->sMemDesc.pvCpuVAddr)[uiPTIndex] &= ~(psConfig->uiValidEnMask);
			}
			else
			{
				eError = PVRSRV_ERROR_MMU_CONFIG_IS_WRONG;
				PVR_DPF((PVR_DBG_ERROR, "Cannot change page table entries due to wrong configuration"));
				goto e_exit;
			}
		}

#if defined(PDUMP)
		PMR_PDumpSymbolicAddr(psPMR, i<<uiLog2PageSize,
		                      sizeof(aszMemspaceName), &aszMemspaceName[0],
		                      sizeof(aszSymbolicAddress), &aszSymbolicAddress[0],
		                      &uiSymbolicAddrOffset,
		                      &uiNextSymName);

		PDumpMMUDumpPxEntries(MMU_LEVEL_1,
		                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
		                      psLevel->sMemDesc.pvCpuVAddr,
		                      psLevel->sMemDesc.sDevPAddr,
		                      uiPTIndex,
		                      1,
		                      aszMemspaceName,
		                      aszSymbolicAddress,
		                      uiSymbolicAddrOffset,
		                      psConfig->uiBytesPerEntry,
		                      psConfig->uiAddrLog2Align,
		                      psConfig->uiAddrShift,
		                      psConfig->uiAddrMask,
		                      psConfig->uiProtMask,
		                      psConfig->uiValidEnMask,
		                      0,
		                      psMMUContext->psDevAttrs->eMMUType);
#endif /*PDUMP*/

		sDevVAddr.uiAddr += uiPageSize;
		i++;

		/* Calculate PT index and get new table descriptor */
		if (uiPTIndex < (psDevVAddrConfig->uiNumEntriesPT - 1) && (i != uiNumPages))
		{
			uiPTIndex++;
		}
		else
		{

			eError = psMMUContext->psDevNode->pfnDevPxClean(psMMUContext->psDevNode,
			                                                &psLevel->sMemDesc.psMapping->sMemHandle,
			                                                uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
			                                                (uiPTIndex+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
			if (eError != PVRSRV_OK)
				goto e_exit;

			_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
			               &psLevel, &uiPTIndex);
			uiFlushStart = uiPTIndex;
		}
	}

	e_exit:

	_MMU_PutPTConfig(psMMUContext, hPriv);

	/* Flush TLB for PTs*/
	psMMUContext->psDevNode->pfnMMUCacheInvalidate(psMMUContext->psDevNode,
	                                               psMMUContext->hDevData,
	                                               MMU_LEVEL_1,
	                                               !bMakeValid);

	PVR_ASSERT(eError == PVRSRV_OK);
	return eError;
}


/*
	MMU_AcquireBaseAddr
 */
PVRSRV_ERROR
MMU_AcquireBaseAddr(MMU_CONTEXT *psMMUContext, IMG_DEV_PHYADDR *psPhysAddr)
{
	if (!psMMUContext)
	{
		psPhysAddr->uiAddr = 0;
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	*psPhysAddr = psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr;
	return PVRSRV_OK;
}

/*
	MMU_ReleaseBaseAddr
 */
void
MMU_ReleaseBaseAddr(MMU_CONTEXT *psMMUContext)
{
	PVR_UNREFERENCED_PARAMETER(psMMUContext);
}

/*
	MMU_SetDeviceData
 */
void MMU_SetDeviceData(MMU_CONTEXT *psMMUContext, IMG_HANDLE hDevData)
{
	psMMUContext->hDevData = hDevData;
}

#if defined(SUPPORT_GPUVIRT_VALIDATION)
/*
    MMU_SetOSid, MMU_GetOSid
 */

void MMU_SetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32OSid, IMG_UINT32 ui32OSidReg, IMG_BOOL bOSidAxiProt)
{
	psMMUContext->ui32OSid     = ui32OSid;
	psMMUContext->ui32OSidReg  = ui32OSidReg;
	psMMUContext->bOSidAxiProt = bOSidAxiProt;

	return;
}

void MMU_GetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 *pui32OSid, IMG_UINT32 *pui32OSidReg, IMG_BOOL *pbOSidAxiProt)
{
	*pui32OSid     = psMMUContext->ui32OSid;
	*pui32OSidReg  = psMMUContext->ui32OSidReg;
	*pbOSidAxiProt = psMMUContext->bOSidAxiProt;

	return;
}

#endif

/*
	MMU_CheckFaultAddress
 */
void MMU_CheckFaultAddress(MMU_CONTEXT *psMMUContext,
                           IMG_DEV_VIRTADDR *psDevVAddr,
                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
                           void *pvDumpDebugFile,
                           MMU_FAULT_DATA *psOutFaultData)
{
	/* Ideally the RGX defs should be via callbacks, but the function is only called from RGX. */
#define MMU_VALID_STR(entry,level) \
		(apszMMUValidStr[((((entry)&(RGX_MMUCTRL_##level##_DATA_ENTRY_PENDING_EN))!=0) << 1)| \
		                 ((((entry)&(RGX_MMUCTRL_##level##_DATA_VALID_EN))!=0) << 0)])
	static const IMG_PCHAR apszMMUValidStr[1<<2] = {/*--*/ "not valid",
			/*-V*/ "valid",
			/*P-*/ "pending",
	/*PV*/ "inconsistent (pending and valid)"};
	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
	MMU_LEVEL	eMMULevel = psDevAttrs->eTopLevel;
	const MMU_PxE_CONFIG *psConfig;
	const MMU_PxE_CONFIG *psMMUPDEConfig;
	const MMU_PxE_CONFIG *psMMUPTEConfig;
	const MMU_DEVVADDR_CONFIG *psMMUDevVAddrConfig;
	IMG_HANDLE hPriv;
	MMU_Levelx_INFO *psLevel = NULL;
	PVRSRV_ERROR eError;
	IMG_UINT64 uiIndex;
	IMG_UINT32 ui32PCIndex;
	IMG_UINT32 ui32PDIndex;
	IMG_UINT32 ui32PTIndex;
	IMG_UINT32 ui32Log2PageSize;
	MMU_FAULT_DATA sMMUFaultData = {0};
	MMU_LEVEL_DATA *psMMULevelData;

	OSLockAcquire(psMMUContext->hLock);

	/*
		At this point we don't know the page size so assume it's 4K.
		When we get the PD level (MMU_LEVEL_2) we can check to see
		if this assumption is correct.
	 */
	eError = psDevAttrs->pfnGetPageSizeConfiguration(12,
	                                                 &psMMUPDEConfig,
	                                                 &psMMUPTEConfig,
	                                                 &psMMUDevVAddrConfig,
	                                                 &hPriv);
	if (eError != PVRSRV_OK)
	{
		PVR_LOG(("Failed to get the page size info for log2 page sizeof 12"));
	}

	psLevel = &psMMUContext->sBaseLevelInfo;
	psConfig = psDevAttrs->psBaseConfig;

	sMMUFaultData.eTopLevel = psDevAttrs->eTopLevel;
	sMMUFaultData.eType = MMU_FAULT_TYPE_NON_PM;


	for (; eMMULevel > MMU_LEVEL_0; eMMULevel--)
	{
		if (eMMULevel == MMU_LEVEL_3)
		{
			/* Determine the PC index */
			uiIndex = psDevVAddr->uiAddr & psDevAttrs->psTopLevelDevVAddrConfig->uiPCIndexMask;
			uiIndex = uiIndex >> psDevAttrs->psTopLevelDevVAddrConfig->uiPCIndexShift;
			ui32PCIndex = (IMG_UINT32) uiIndex;
			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PCIndex));

			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_3];
			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
			psMMULevelData->ui32Index = ui32PCIndex;

			if (ui32PCIndex >= psLevel->ui32NumOfEntries)
			{
				PVR_DUMPDEBUG_LOG("PC index (%d) out of bounds (%d)", ui32PCIndex, psLevel->ui32NumOfEntries);
				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
				break;
			}

			if (psConfig->uiBytesPerEntry == 4)
			{
				IMG_UINT32 *pui32Ptr = psLevel->sMemDesc.pvCpuVAddr;

				PVR_DUMPDEBUG_LOG("PCE for index %d = 0x%08x and is %s",
				                  ui32PCIndex,
				                  pui32Ptr[ui32PCIndex],
				                  MMU_VALID_STR(pui32Ptr[ui32PCIndex], PC));

				psMMULevelData->ui64Address = pui32Ptr[ui32PCIndex];
				psMMULevelData->psDebugStr  = MMU_VALID_STR(pui32Ptr[ui32PCIndex], PC);
			}
			else
			{
				IMG_UINT64 *pui64Ptr = psLevel->sMemDesc.pvCpuVAddr;

				PVR_DUMPDEBUG_LOG("PCE for index %d = 0x%016" IMG_UINT64_FMTSPECx " and is %s",
				                  ui32PCIndex,
				                  pui64Ptr[ui32PCIndex],
				                  MMU_VALID_STR(pui64Ptr[ui32PCIndex], PC));

				psMMULevelData->ui64Address = pui64Ptr[ui32PCIndex];
				psMMULevelData->psDebugStr  = MMU_VALID_STR(pui64Ptr[ui32PCIndex], PC);
			}

			psLevel = psLevel->apsNextLevel[ui32PCIndex];
			if (!psLevel)
			{
				break;
			}
			psConfig = psMMUPDEConfig;
		}


		if (eMMULevel == MMU_LEVEL_2)
		{
			/* Determine the PD index */
			uiIndex = psDevVAddr->uiAddr & psDevAttrs->psTopLevelDevVAddrConfig->uiPDIndexMask;
			uiIndex = uiIndex >> psDevAttrs->psTopLevelDevVAddrConfig->uiPDIndexShift;
			ui32PDIndex = (IMG_UINT32) uiIndex;
			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PDIndex));

			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_2];
			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
			psMMULevelData->ui32Index = ui32PDIndex;

			if (ui32PDIndex >= psLevel->ui32NumOfEntries)
			{
				PVR_DUMPDEBUG_LOG("PD index (%d) out of bounds (%d)", ui32PDIndex, psLevel->ui32NumOfEntries);
				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
				break;
			}

			if (psConfig->uiBytesPerEntry == 4)
			{
				IMG_UINT32 *pui32Ptr = psLevel->sMemDesc.pvCpuVAddr;

				/* FIXME: MMU_VALID_STR doesn't work here because
				 *        RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_EN is wider than
				 *        32bits */

				PVR_DUMPDEBUG_LOG("PDE for index %d = 0x%08x and is %s",
				                  ui32PDIndex,
				                  pui32Ptr[ui32PDIndex],
				                  MMU_VALID_STR(pui32Ptr[ui32PDIndex], PD));

				psMMULevelData->ui64Address = pui32Ptr[ui32PDIndex];
				psMMULevelData->psDebugStr  = MMU_VALID_STR(pui32Ptr[ui32PDIndex], PD);

				if (psDevAttrs->pfnGetPageSizeFromPDE4(pui32Ptr[ui32PDIndex], &ui32Log2PageSize) != PVRSRV_OK)
				{
					PVR_LOG(("Failed to get the page size from the PDE"));
				}
			}
			else
			{
				IMG_UINT64 *pui64Ptr = psLevel->sMemDesc.pvCpuVAddr;

				PVR_DUMPDEBUG_LOG("PDE for index %d = 0x%016" IMG_UINT64_FMTSPECx " and is %s",
				                  ui32PDIndex,
				                  pui64Ptr[ui32PDIndex],
				                  MMU_VALID_STR(pui64Ptr[ui32PDIndex], PD));

				psMMULevelData->ui64Address = pui64Ptr[ui32PDIndex];
				psMMULevelData->psDebugStr  = MMU_VALID_STR(pui64Ptr[ui32PDIndex], PD);

				if (psDevAttrs->pfnGetPageSizeFromPDE8(pui64Ptr[ui32PDIndex], &ui32Log2PageSize) != PVRSRV_OK)
				{
					PVR_LOG(("Failed to get the page size from the PDE"));
				}
			}

			/*
					We assumed the page size was 4K, now we have the actual size
					from the PDE we can confirm if our assumption was correct.
					Until now it hasn't mattered as the PC and PD are the same
					regardless of the page size
			 */
			if (ui32Log2PageSize != 12)
			{
				/* Put the 4K page size data */
				psDevAttrs->pfnPutPageSizeConfiguration(hPriv);

				/* Get the correct size data */
				eError = psDevAttrs->pfnGetPageSizeConfiguration(ui32Log2PageSize,
				                                                 &psMMUPDEConfig,
				                                                 &psMMUPTEConfig,
				                                                 &psMMUDevVAddrConfig,
				                                                 &hPriv);
				if (eError != PVRSRV_OK)
				{
					PVR_LOG(("Failed to get the page size info for log2 page sizeof %d", ui32Log2PageSize));
					break;
				}
			}
			psLevel = psLevel->apsNextLevel[ui32PDIndex];
			if (!psLevel)
			{
				break;
			}
			psConfig = psMMUPTEConfig;
		}


		if (eMMULevel == MMU_LEVEL_1)
		{
			/* Determine the PT index */
			uiIndex = psDevVAddr->uiAddr & psMMUDevVAddrConfig->uiPTIndexMask;
			uiIndex = uiIndex >> psMMUDevVAddrConfig->uiPTIndexShift;
			ui32PTIndex = (IMG_UINT32) uiIndex;
			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PTIndex));

			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_1];
			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
			psMMULevelData->ui32Index = ui32PTIndex;

			if (ui32PTIndex >= psLevel->ui32NumOfEntries)
			{
				PVR_DUMPDEBUG_LOG("PT index (%d) out of bounds (%d)", ui32PTIndex, psLevel->ui32NumOfEntries);
				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
				break;
			}

			if (psConfig->uiBytesPerEntry == 4)
			{
				IMG_UINT32 *pui32Ptr = psLevel->sMemDesc.pvCpuVAddr;

				PVR_DUMPDEBUG_LOG("PTE for index %d = 0x%08x and is %s",
				                  ui32PTIndex,
				                  pui32Ptr[ui32PTIndex],
				                  MMU_VALID_STR(pui32Ptr[ui32PTIndex], PT));

				psMMULevelData->ui64Address = pui32Ptr[ui32PTIndex];
				psMMULevelData->psDebugStr  = MMU_VALID_STR(pui32Ptr[ui32PTIndex], PT);
			}
			else
			{
				IMG_UINT64 *pui64Ptr = psLevel->sMemDesc.pvCpuVAddr;

				PVR_DUMPDEBUG_LOG("PTE for index %d = 0x%016" IMG_UINT64_FMTSPECx " and is %s",
				                  ui32PTIndex,
				                  pui64Ptr[ui32PTIndex],
				                  MMU_VALID_STR(pui64Ptr[ui32PTIndex], PT));

				psMMULevelData->ui64Address = pui64Ptr[ui32PTIndex];
				psMMULevelData->psDebugStr  = MMU_VALID_STR(pui64Ptr[ui32PTIndex], PT);
			}
			goto e1;
		}

		PVR_LOG(("Unsupported MMU setup"));
	}

	e1:
	/* Put the page size data back */
	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
	OSLockRelease(psMMUContext->hLock);

	if (psOutFaultData)
	{
		*psOutFaultData = sMMUFaultData;
	}
}

IMG_BOOL MMU_IsVDevAddrValid(MMU_CONTEXT *psMMUContext,
                             IMG_UINT32 uiLog2PageSize,
                             IMG_DEV_VIRTADDR sDevVAddr)
{
	MMU_Levelx_INFO *psLevel = NULL;
	const MMU_PxE_CONFIG *psConfig;
	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
	IMG_HANDLE hPriv = NULL;
	IMG_UINT32 uiIndex = 0;
	IMG_BOOL bStatus = IMG_FALSE;

	_MMU_GetPTConfig(psMMUContext, uiLog2PageSize, &psConfig, &hPriv, &psDevVAddrConfig);

	OSLockAcquire(psMMUContext->hLock);

	switch (psMMUContext->psDevAttrs->eTopLevel)
	{
		case MMU_LEVEL_3:
			uiIndex = _CalcPCEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
			psLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiIndex];
			if (psLevel == NULL)
				break;

			__fallthrough;
		case MMU_LEVEL_2:
			uiIndex = _CalcPDEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);

			if (psLevel != NULL)
				psLevel = psLevel->apsNextLevel[uiIndex];
			else
				psLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiIndex];

			if (psLevel == NULL)
				break;

			__fallthrough;
		case MMU_LEVEL_1:
			uiIndex = _CalcPTEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);

			if (psLevel == NULL)
				psLevel = &psMMUContext->sBaseLevelInfo;

			bStatus = ((IMG_UINT64 *) psLevel->sMemDesc.pvCpuVAddr)[uiIndex]
			                                                        & psConfig->uiValidEnMask;
			break;
		default:
			PVR_LOG(("MMU_IsVDevAddrValid: Unsupported MMU setup"));
			break;
	}

	OSLockRelease(psMMUContext->hLock);

	_MMU_PutPTConfig(psMMUContext, hPriv);

	return bStatus;
}

#if defined(PDUMP)
IMG_CHAR *MMU_GetPxPDumpMemSpaceName(MMU_CONTEXT *psMMUContext)
{
	return psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName;
}

/*
	MMU_ContextDerivePCPDumpSymAddr
 */
PVRSRV_ERROR MMU_ContextDerivePCPDumpSymAddr(MMU_CONTEXT *psMMUContext,
                                             IMG_CHAR *pszPDumpSymbolicNameBuffer,
                                             size_t uiPDumpSymbolicNameBufferSize)
{
	size_t uiCount;
	IMG_UINT64 ui64PhysAddr;
	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psDevNode->sDevId;

	if (!psMMUContext->sBaseLevelInfo.sMemDesc.bValid)
	{
		/* We don't have any allocations.  You're not allowed to ask
           for the page catalogue base address until you've made at
           least one allocation */
		return PVRSRV_ERROR_MMU_API_PROTOCOL_ERROR;
	}

	ui64PhysAddr = (IMG_UINT64)psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr.uiAddr;

	PVR_ASSERT(uiPDumpSymbolicNameBufferSize >= (IMG_UINT32)(21 + OSStringLength(psDevId->pszPDumpDevName)));

	/* Page table Symbolic Name is formed from page table phys addr
       prefixed with MMUPT_. */

	uiCount = OSSNPrintf(pszPDumpSymbolicNameBuffer,
	                     uiPDumpSymbolicNameBufferSize,
	                     ":%s:%s%016"IMG_UINT64_FMTSPECX,
	                     psDevId->pszPDumpDevName,
	                     psMMUContext->sBaseLevelInfo.sMemDesc.bValid?"MMUPC_":"XXX",
	                    		 ui64PhysAddr);

	if (uiCount + 1 > uiPDumpSymbolicNameBufferSize)
	{
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	return PVRSRV_OK;
}

/*
	MMU_PDumpWritePageCatBase
 */
PVRSRV_ERROR
MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
                          const IMG_CHAR *pszSpaceName,
                          IMG_DEVMEM_OFFSET_T uiOffset,
                          IMG_UINT32 ui32WordSize,
                          IMG_UINT32 ui32AlignShift,
                          IMG_UINT32 ui32Shift,
                          PDUMP_FLAGS_T uiPdumpFlags)
{
	PVRSRV_ERROR eError;
	IMG_CHAR aszPageCatBaseSymbolicAddr[100];
	const IMG_CHAR *pszPDumpDevName = psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName;

	eError = MMU_ContextDerivePCPDumpSymAddr(psMMUContext,
	                                         &aszPageCatBaseSymbolicAddr[0],
	                                         sizeof(aszPageCatBaseSymbolicAddr));
	if (eError == PVRSRV_OK)
	{
		eError = PDumpWriteSymbAddress(pszSpaceName,
		                               uiOffset,
		                               aszPageCatBaseSymbolicAddr,
		                               0, /* offset -- Could be non-zero for var. pgsz */
		                               pszPDumpDevName,
		                               ui32WordSize,
		                               ui32AlignShift,
		                               ui32Shift,
		                               uiPdumpFlags | PDUMP_FLAGS_CONTINUOUS);
	}

	return eError;
}

/*
	MMU_AcquirePDumpMMUContext
 */
PVRSRV_ERROR MMU_AcquirePDumpMMUContext(MMU_CONTEXT *psMMUContext,
                                        IMG_UINT32 *pui32PDumpMMUContextID)
{
	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psDevNode->sDevId;

	if (!psMMUContext->ui32PDumpContextIDRefCount)
	{
		PDUMP_MMU_ALLOC_MMUCONTEXT(psDevId->pszPDumpDevName,
		                           psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr,
		                           psMMUContext->psDevAttrs->eMMUType,
		                           &psMMUContext->uiPDumpContextID);
	}

	psMMUContext->ui32PDumpContextIDRefCount++;
	*pui32PDumpMMUContextID = psMMUContext->uiPDumpContextID;

	return PVRSRV_OK;
}

/*
	MMU_ReleasePDumpMMUContext
 */
PVRSRV_ERROR MMU_ReleasePDumpMMUContext(MMU_CONTEXT *psMMUContext)
{
	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psDevNode->sDevId;

	PVR_ASSERT(psMMUContext->ui32PDumpContextIDRefCount != 0);
	psMMUContext->ui32PDumpContextIDRefCount--;

	if (psMMUContext->ui32PDumpContextIDRefCount == 0)
	{
		PDUMP_MMU_FREE_MMUCONTEXT(psDevId->pszPDumpDevName,
		                          psMMUContext->uiPDumpContextID);
	}

	return PVRSRV_OK;
}
#endif

/******************************************************************************
 End of file (mmu_common.c)
 ******************************************************************************/