drivers/misc/mediatek/gpu/gpu_rgx/m1.11_5516664/system/dma_support.c - linux-mtk - Git at Google

 /*************************************************************************/ /*!
 @File           dma_support.c
 @Title          System DMA support
 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
 @Description    Provides a contiguous memory allocator (i.e. DMA allocator);
                 APIs are used for allocation/ioremapping (DMA/PA <-> CPU/VA)
 @License        Dual MIT/GPLv2

 The contents of this file are subject to the MIT license as set out below.

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:

 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.

 Alternatively, the contents of this file may be used under the terms of
 the GNU General Public License Version 2 ("GPL") in which case the provisions
 of GPL are applicable instead of those above.

 If you wish to allow use of your version of this file only under the terms of
 GPL, and not to allow others to use your version of this file under the terms
 of the MIT license, indicate your decision by deleting the provisions above
 and replace them with the notice and other provisions required by GPL as set
 out in the file called "GPL-COPYING" included in this distribution. If you do
 not delete the provisions above, a recipient may use your version of this file
 under the terms of either the MIT license or GPL.

 This License is also included in this distribution in the file called
 "MIT-COPYING".

 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */ /**************************************************************************/
 #include <linux/mm.h>
 #include <asm/page.h>
 #include <linux/device.h>
 #include <linux/highmem.h>
 #include <linux/vmalloc.h>
 #include <linux/dma-mapping.h>
 #include <asm-generic/getorder.h>

 #include "allocmem.h"
 #include "dma_support.h"
 #include "kernel_compatibility.h"

 #define DMA_MAX_IOREMAP_ENTRIES 2
 static IMG_BOOL gbEnableDmaIoRemapping = IMG_FALSE;
 static DMA_ALLOC gsDmaIoRemapArray[DMA_MAX_IOREMAP_ENTRIES] = {{0}};

 static void*
 SysDmaAcquireKernelAddress(struct page *psPage, IMG_UINT64 ui64Size, void *pvOSDevice)
 {
 	IMG_UINT32 uiIdx;
 	PVRSRV_ERROR eError;
 	void *pvVirtAddr = NULL;
 	IMG_UINT32 ui32PgCount = (IMG_UINT32)(ui64Size >> OSGetPageShift());
 	PVRSRV_DEVICE_NODE *psDevNode = OSAllocZMemNoStats(sizeof(*psDevNode));
 	PVRSRV_DEVICE_CONFIG *psDevConfig = OSAllocZMemNoStats(sizeof(*psDevConfig));
 	struct page **pagearray = OSAllocZMemNoStats(ui32PgCount * sizeof(struct page *));
 #if defined(CONFIG_ARM64)
 	pgprot_t prot = pgprot_writecombine(PAGE_KERNEL);
 #else
 	pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
 #endif

 	/* Validate all required dynamic tmp buffer allocations */
 	if (psDevNode == NULL || psDevConfig == NULL || pagearray == NULL)
 	{
 		if (psDevNode)
 		{
 			OSFreeMem(psDevNode);
 		}

 		if (psDevConfig)
 		{
 			OSFreeMem(psDevConfig);
 		}

 		if (pagearray)
 		{
 			OSFreeMem(pagearray);
 		}

 		goto e0;
 	}

 	/* Fake psDevNode->psDevConfig->pvOSDevice */
 	psDevConfig->pvOSDevice = pvOSDevice;
 	psDevNode->psDevConfig = psDevConfig;

 	/* Evict any page data contents from d-cache */
 #if defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
 	/* ARM platforms do not support a global flush. */
 	eError = PVRSRV_ERROR_RETRY;
 #else
 	eError = OSCPUOperation(PVRSRV_CACHE_OP_FLUSH);
 #endif
 	for (uiIdx = 0; uiIdx  < ui32PgCount; uiIdx++)
 	{
 		/* Prepare array required for vmap */
 		pagearray[uiIdx] = &psPage[uiIdx];

 		if (eError != PVRSRV_OK)
 		{
 #if defined(CONFIG_64BIT)
 			void *pvVirtStart = kmap(&psPage[uiIdx]);
 			void *pvVirtEnd = pvVirtStart + ui64Size;
 			IMG_CPU_PHYADDR sCPUPhysStart = {page_to_phys(&psPage[uiIdx])};
 			IMG_CPU_PHYADDR sCPUPhysEnd = {sCPUPhysStart.uiAddr + ui64Size};
 			/* all pages have a kernel linear address, flush entire range */
 #else
 			void *pvVirtStart = kmap(&psPage[uiIdx]);
 			void *pvVirtEnd = pvVirtStart + PAGE_SIZE;
 			IMG_CPU_PHYADDR sCPUPhysStart = {page_to_phys(&psPage[uiIdx])};
 			IMG_CPU_PHYADDR sCPUPhysEnd = {sCPUPhysStart.uiAddr + PAGE_SIZE};
 			/* pages might be from HIGHMEM, need to kmap/flush per page */
 #endif

 			/* Fallback to range-based d-cache flush */
 			OSCPUCacheInvalidateRangeKM(psDevNode,
 										pvVirtStart, pvVirtEnd,
 										sCPUPhysStart, sCPUPhysEnd);

 #if defined(CONFIG_64BIT)
 			eError = PVRSRV_OK;
 #else
 			kunmap(&psPage[uiIdx]);
 #endif
 		}
 	}

 	/* Remap pages into VMALLOC space */
 #if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
 	pvVirtAddr = vmap(pagearray, ui32PgCount, VM_READ | VM_WRITE, prot);
 #else
 	pvVirtAddr = vm_map_ram(pagearray, ui32PgCount, -1, prot);
 #endif

 	/* Clean-up tmp buffers */
 	OSFreeMem(psDevConfig);
 	OSFreeMem(psDevNode);
 	OSFreeMem(pagearray);

 e0:
 	return pvVirtAddr;
 }

 static void SysDmaReleaseKernelAddress(void *pvVirtAddr, IMG_UINT64 ui64Size)
 {
 #if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
 	vunmap(pvVirtAddr);
 #else
 	vm_unmap_ram(pvVirtAddr, ui64Size >> OSGetPageShift());
 #endif
 }

 /*!
 ******************************************************************************
  @Function			SysDmaAllocMem

  @Description 		Allocates physically contiguous memory

  @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
 									error code
  ******************************************************************************/
 PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc)
 {
 	PVRSRV_ERROR eError = PVRSRV_OK;
 	struct device *psDev;
 	struct page *psPage;
 	size_t uiSize;

 	if (psDmaAlloc == NULL ||
 		psDmaAlloc->hHandle ||
 		psDmaAlloc->pvVirtAddr ||
 		psDmaAlloc->ui64Size == 0 ||
 		psDmaAlloc->sBusAddr.uiAddr ||
 		psDmaAlloc->pvOSDevice == NULL)
 	{
 		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
 		return PVRSRV_ERROR_INVALID_PARAMS;
 	}

 	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
 	psDev = (struct device *)psDmaAlloc->pvOSDevice;

 #if !(defined(CONFIG_L4) || defined(PVR_LINUX_PHYSMEM_SUPPRESS_DMA_AC))
 	psDmaAlloc->hHandle = dma_alloc_coherent(psDev, uiSize, (dma_addr_t *)&psDmaAlloc->sBusAddr.uiAddr, GFP_KERNEL);
 #endif
 	if (psDmaAlloc->hHandle)
 	{
 		psDmaAlloc->pvVirtAddr = psDmaAlloc->hHandle;

 		PVR_DPF((PVR_DBG_MESSAGE,
 				"Allocated DMA buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
 				psDmaAlloc->pvVirtAddr,
 				psDmaAlloc->sBusAddr.uiAddr,
 				uiSize));
 	}
 	else if ((psPage = alloc_pages(GFP_KERNEL, get_order(uiSize))))
 	{
 #if defined(CONFIG_L4)
 		/* L4 is a para-virtualized environment, the PFN space is a virtual space and not physical space */
 		psDmaAlloc->sBusAddr.uiAddr = l4x_virt_to_phys((void*)((unsigned long)page_to_pfn(psPage) << PAGE_SHIFT));
 #else
 		psDmaAlloc->sBusAddr.uiAddr = dma_map_page(psDev, psPage, 0, uiSize, DMA_BIDIRECTIONAL);
 		if (dma_mapping_error(psDev, psDmaAlloc->sBusAddr.uiAddr))
 		{
 			PVR_DPF((PVR_DBG_ERROR,
 					"dma_map_page() failed, page 0x%p order %d",
 					psPage,
 					get_order(uiSize)));
 			__free_pages(psPage, get_order(uiSize));
 			goto e0;
 		}
 		psDmaAlloc->psPage = psPage;
 #endif

 		psDmaAlloc->pvVirtAddr = SysDmaAcquireKernelAddress(psPage, uiSize, psDmaAlloc->pvOSDevice);
 		if (! psDmaAlloc->pvVirtAddr)
 		{
 			PVR_DPF((PVR_DBG_ERROR,
 					"SysDmaAcquireKernelAddress() failed, page 0x%p order %d",
 					psPage,
 					get_order(uiSize)));
 #if !defined(CONFIG_L4)
 			dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
 #endif
 			__free_pages(psPage, get_order(uiSize));
 			goto e0;
 		}

 		PVR_DPF((PVR_DBG_MESSAGE,
 				"Allocated contiguous buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
 				psDmaAlloc->pvVirtAddr,
 				psDmaAlloc->sBusAddr.uiAddr,
 				uiSize));
 	}
 	else
 	{
 		PVR_DPF((PVR_DBG_ERROR,	"Unable to allocate contiguous buffer, size: 0x"IMG_SIZE_FMTSPECX, uiSize));
 		eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
 	}

 e0:
 	PVR_LOGR_IF_FALSE((psDmaAlloc->pvVirtAddr), "DMA/CMA allocation failed", PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES);
 	return eError;
 }

 /*!
 ******************************************************************************
  @Function			SysDmaFreeMem

  @Description 		Free physically contiguous memory

  @Return			void
  ******************************************************************************/
 void SysDmaFreeMem(DMA_ALLOC *psDmaAlloc)
 {
 	size_t uiSize;
 	struct device *psDev;

 	if (psDmaAlloc == NULL ||
 		psDmaAlloc->ui64Size == 0 ||
 		psDmaAlloc->pvOSDevice == NULL ||
 		psDmaAlloc->pvVirtAddr == NULL ||
 		psDmaAlloc->sBusAddr.uiAddr == 0)
 	{
 		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
 		return;
 	}

 	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
 	psDev = (struct device *)psDmaAlloc->pvOSDevice;

 	if (psDmaAlloc->pvVirtAddr != psDmaAlloc->hHandle)
 	{
 		SysDmaReleaseKernelAddress(psDmaAlloc->pvVirtAddr, uiSize);
 	}

 	if (! psDmaAlloc->hHandle)
 	{
 		struct page *psPage;
 #if defined(CONFIG_L4)
 		psPage = pfn_to_page((unsigned long)l4x_phys_to_virt(psDmaAlloc->sBusAddr.uiAddr) >> PAGE_SHIFT);
 #else
 		dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
 		psPage = psDmaAlloc->psPage;
 #endif
 		__free_pages(psPage, get_order(uiSize));
 		return;
 	}

 	dma_free_coherent(psDev, uiSize, psDmaAlloc->hHandle, (dma_addr_t )psDmaAlloc->sBusAddr.uiAddr);
 }

 /*!
 ******************************************************************************
  @Function			SysDmaRegisterForIoRemapping

  @Description 		Registers DMA_ALLOC for manual I/O remapping

  @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
 									error code
  ******************************************************************************/
 PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
 {
 	size_t uiSize;
 	IMG_UINT32 ui32Idx;
 	IMG_BOOL bTabEntryFound = IMG_TRUE;
 	PVRSRV_ERROR eError = PVRSRV_ERROR_TOO_FEW_BUFFERS;

 	if (psDmaAlloc == NULL ||
 		psDmaAlloc->ui64Size == 0 ||
 		psDmaAlloc->pvOSDevice == NULL ||
 		psDmaAlloc->pvVirtAddr == NULL ||
 		psDmaAlloc->sBusAddr.uiAddr == 0)
 	{
 		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
 		return PVRSRV_ERROR_INVALID_PARAMS;
 	}

 	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);

 	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
 	{
 		/* Check if an I/O remap entry exists for remapping */
 		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == NULL)
 		{
 			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == 0);
 			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].ui64Size == 0);
 			break;
 		}
 	}

 	if (ui32Idx >= DMA_MAX_IOREMAP_ENTRIES)
 	{
 		bTabEntryFound = IMG_FALSE;
 	}

 	if (bTabEntryFound)
 	{
 		IMG_BOOL bSameVAddr, bSamePAddr, bSameSize;

 		bSamePAddr = gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == psDmaAlloc->sBusAddr.uiAddr;
 		bSameVAddr = gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr;
 		bSameSize = gsDmaIoRemapArray[ui32Idx].ui64Size == uiSize;

 		if (bSameVAddr)
 		{
 			if (bSamePAddr && bSameSize)
 			{
 				eError = PVRSRV_OK;
 			}
 			else
 			{
 				eError = PVRSRV_ERROR_ALREADY_EXISTS;
 			}
 		}
 		else
 		{
 			PVR_ASSERT(bSamePAddr == IMG_FALSE);

 			gsDmaIoRemapArray[ui32Idx].ui64Size = uiSize;
 			gsDmaIoRemapArray[ui32Idx].sBusAddr = psDmaAlloc->sBusAddr;
 			gsDmaIoRemapArray[ui32Idx].pvVirtAddr = psDmaAlloc->pvVirtAddr;

 			PVR_DPF((PVR_DBG_MESSAGE,
 					"DMA: register I/O remap: "
 					"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
 					psDmaAlloc->pvVirtAddr,
 					psDmaAlloc->sBusAddr.uiAddr,
 					uiSize));

 			gbEnableDmaIoRemapping = IMG_TRUE;
 			eError = PVRSRV_OK;
 		}
 	}

 	return eError;
 }

 /*!
 ******************************************************************************
  @Function			SysDmaDeregisterForIoRemapping

  @Description 		Deregisters DMA_ALLOC from manual I/O remapping

  @Return			void
  ******************************************************************************/
 void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
 {
 	size_t uiSize;
 	IMG_UINT32 ui32Idx;

 	if (psDmaAlloc == NULL ||
 		psDmaAlloc->ui64Size == 0 ||
 		psDmaAlloc->pvOSDevice == NULL ||
 		psDmaAlloc->pvVirtAddr == NULL ||
 		psDmaAlloc->sBusAddr.uiAddr == 0)
 	{
 		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
 		return;
 	}

 	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);

 	/* Remove specified entries from list of I/O remap entries */
 	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
 	{
 		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr)
 		{
 			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr = 0;
 			gsDmaIoRemapArray[ui32Idx].pvVirtAddr = NULL;
 			gsDmaIoRemapArray[ui32Idx].ui64Size =  0;

 			PVR_DPF((PVR_DBG_MESSAGE,
 					"DMA: deregister I/O remap: "
 					"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
 					psDmaAlloc->pvVirtAddr,
 					psDmaAlloc->sBusAddr.uiAddr,
 					uiSize));

 			break;
 		}
 	}

 	/* Check if no other I/O remap entries exists for remapping */
 	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
 	{
 		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr != NULL)
 		{
 			break;
 		}
 	}

 	if (ui32Idx == DMA_MAX_IOREMAP_ENTRIES)
 	{
 		/* No entries found so disable remapping */
 		gbEnableDmaIoRemapping = IMG_FALSE;
 	}
 }

 /*!
 ******************************************************************************
  @Function			SysDmaDevPAddrToCpuVAddr

  @Description 		Maps a DMA_ALLOC physical address to CPU virtual address

  @Return			IMG_CPU_VIRTADDR on success. Otherwise, a NULL
  ******************************************************************************/
 IMG_CPU_VIRTADDR SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size)
 {
 	IMG_CPU_VIRTADDR pvDMAVirtAddr = NULL;
 	DMA_ALLOC *psHeapDmaAlloc;
 	IMG_UINT32 ui32Idx;

 	if (gbEnableDmaIoRemapping == IMG_FALSE)
 	{
 		return pvDMAVirtAddr;
 	}

 	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
 	{
 		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
 		if (psHeapDmaAlloc->sBusAddr.uiAddr && uiAddr >= psHeapDmaAlloc->sBusAddr.uiAddr)
 		{
 			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
 			IMG_UINT64 uiOffset = uiAddr - psHeapDmaAlloc->sBusAddr.uiAddr;

 			if (uiOffset < uiSpan)
 			{
 				PVR_ASSERT((uiOffset+ui64Size-1) < uiSpan);
 				pvDMAVirtAddr = psHeapDmaAlloc->pvVirtAddr + uiOffset;

 				PVR_DPF((PVR_DBG_MESSAGE,
 					"DMA: remap: PA: 0x%llx => VA: 0x%p",
 					uiAddr, pvDMAVirtAddr));

 				break;
 			}
 		}
 	}

 	return pvDMAVirtAddr;
 }

 /*!
 ******************************************************************************
  @Function			SysDmaCpuVAddrToDevPAddr

  @Description 		Maps a DMA_ALLOC CPU virtual address to physical address

  @Return			Non-zero value on success. Otherwise, a 0
  ******************************************************************************/
 IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr)
 {
 	IMG_UINT64 uiAddr = 0;
 	DMA_ALLOC *psHeapDmaAlloc;
 	IMG_UINT32 ui32Idx;

 	if (gbEnableDmaIoRemapping == IMG_FALSE)
 	{
 		return uiAddr;
 	}

 	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
 	{
 		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
 		if (psHeapDmaAlloc->pvVirtAddr && pvDMAVirtAddr >= psHeapDmaAlloc->pvVirtAddr)
 		{
 			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
 			IMG_UINT64 uiOffset = pvDMAVirtAddr - psHeapDmaAlloc->pvVirtAddr;

 			if (uiOffset < uiSpan)
 			{
 				uiAddr = psHeapDmaAlloc->sBusAddr.uiAddr + uiOffset;

 				PVR_DPF((PVR_DBG_MESSAGE,
 					"DMA: remap: VA: 0x%p => PA: 0x%llx",
 					pvDMAVirtAddr, uiAddr));

 				break;
 			}
 		}
 	}

 	return uiAddr;
 }

 /******************************************************************************
  End of file (dma_support.c)
 ******************************************************************************/
	/************************************************************************/ /!
	@File dma_support.c
	@Title System DMA support
	@Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved
	@Description Provides a contiguous memory allocator (i.e. DMA allocator);
	APIs are used for allocation/ioremapping (DMA/PA <-> CPU/VA)
	@License Dual MIT/GPLv2

	The contents of this file are subject to the MIT license as set out below.

	Permission is hereby granted, free of charge, to any person obtaining a copy
	of this software and associated documentation files (the "Software"), to deal
	in the Software without restriction, including without limitation the rights
	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	copies of the Software, and to permit persons to whom the Software is
	furnished to do so, subject to the following conditions:

	The above copyright notice and this permission notice shall be included in
	all copies or substantial portions of the Software.

	Alternatively, the contents of this file may be used under the terms of
	the GNU General Public License Version 2 ("GPL") in which case the provisions
	of GPL are applicable instead of those above.

	If you wish to allow use of your version of this file only under the terms of
	GPL, and not to allow others to use your version of this file under the terms
	of the MIT license, indicate your decision by deleting the provisions above
	and replace them with the notice and other provisions required by GPL as set
	out in the file called "GPL-COPYING" included in this distribution. If you do
	not delete the provisions above, a recipient may use your version of this file
	under the terms of either the MIT license or GPL.

	This License is also included in this distribution in the file called
	"MIT-COPYING".

	EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
	PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
	BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
	PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
	COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
	IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	/ /*************************************************************************/
	#include <linux/mm.h>
	#include <asm/page.h>
	#include <linux/device.h>
	#include <linux/highmem.h>
	#include <linux/vmalloc.h>
	#include <linux/dma-mapping.h>
	#include <asm-generic/getorder.h>

	#include "allocmem.h"
	#include "dma_support.h"
	#include "kernel_compatibility.h"

	#define DMA_MAX_IOREMAP_ENTRIES 2
	static IMG_BOOL gbEnableDmaIoRemapping = IMG_FALSE;
	static DMA_ALLOC gsDmaIoRemapArray[DMA_MAX_IOREMAP_ENTRIES] = {{0}};

	static void*
	SysDmaAcquireKernelAddress(struct page psPage, IMG_UINT64 ui64Size, void pvOSDevice)
	{
	IMG_UINT32 uiIdx;
	PVRSRV_ERROR eError;
	void *pvVirtAddr = NULL;
	IMG_UINT32 ui32PgCount = (IMG_UINT32)(ui64Size >> OSGetPageShift());
	PVRSRV_DEVICE_NODE psDevNode = OSAllocZMemNoStats(sizeof(psDevNode));
	PVRSRV_DEVICE_CONFIG psDevConfig = OSAllocZMemNoStats(sizeof(psDevConfig));
	struct page *pagearray = OSAllocZMemNoStats(ui32PgCount sizeof(struct page *));
	#if defined(CONFIG_ARM64)
	pgprot_t prot = pgprot_writecombine(PAGE_KERNEL);
	#else
	pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
	#endif

	/* Validate all required dynamic tmp buffer allocations */
	if (psDevNode == NULL \|\| psDevConfig == NULL \|\| pagearray == NULL)
	{
	if (psDevNode)
	{
	OSFreeMem(psDevNode);
	}

	if (psDevConfig)
	{
	OSFreeMem(psDevConfig);
	}

	if (pagearray)
	{
	OSFreeMem(pagearray);
	}

	goto e0;
	}

	/* Fake psDevNode->psDevConfig->pvOSDevice */
	psDevConfig->pvOSDevice = pvOSDevice;
	psDevNode->psDevConfig = psDevConfig;

	/* Evict any page data contents from d-cache */
	#if defined(__arm__) \|\| defined(__arm64__) \|\| defined(__aarch64__)
	/* ARM platforms do not support a global flush. */
	eError = PVRSRV_ERROR_RETRY;
	#else
	eError = OSCPUOperation(PVRSRV_CACHE_OP_FLUSH);
	#endif
	for (uiIdx = 0; uiIdx < ui32PgCount; uiIdx++)
	{
	/* Prepare array required for vmap */
	pagearray[uiIdx] = &psPage[uiIdx];

	if (eError != PVRSRV_OK)
	{
	#if defined(CONFIG_64BIT)
	void *pvVirtStart = kmap(&psPage[uiIdx]);
	void *pvVirtEnd = pvVirtStart + ui64Size;
	IMG_CPU_PHYADDR sCPUPhysStart = {page_to_phys(&psPage[uiIdx])};
	IMG_CPU_PHYADDR sCPUPhysEnd = {sCPUPhysStart.uiAddr + ui64Size};
	/* all pages have a kernel linear address, flush entire range */
	#else
	void *pvVirtStart = kmap(&psPage[uiIdx]);
	void *pvVirtEnd = pvVirtStart + PAGE_SIZE;
	IMG_CPU_PHYADDR sCPUPhysStart = {page_to_phys(&psPage[uiIdx])};
	IMG_CPU_PHYADDR sCPUPhysEnd = {sCPUPhysStart.uiAddr + PAGE_SIZE};
	/* pages might be from HIGHMEM, need to kmap/flush per page */
	#endif

	/* Fallback to range-based d-cache flush */
	OSCPUCacheInvalidateRangeKM(psDevNode,
	pvVirtStart, pvVirtEnd,
	sCPUPhysStart, sCPUPhysEnd);

	#if defined(CONFIG_64BIT)
	eError = PVRSRV_OK;
	#else
	kunmap(&psPage[uiIdx]);
	#endif
	}
	}

	/* Remap pages into VMALLOC space */
	#if !defined(CONFIG_64BIT) \|\| defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
	pvVirtAddr = vmap(pagearray, ui32PgCount, VM_READ \| VM_WRITE, prot);
	#else
	pvVirtAddr = vm_map_ram(pagearray, ui32PgCount, -1, prot);
	#endif

	/* Clean-up tmp buffers */
	OSFreeMem(psDevConfig);
	OSFreeMem(psDevNode);
	OSFreeMem(pagearray);

	e0:
	return pvVirtAddr;
	}

	static void SysDmaReleaseKernelAddress(void *pvVirtAddr, IMG_UINT64 ui64Size)
	{
	#if !defined(CONFIG_64BIT) \|\| defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
	vunmap(pvVirtAddr);
	#else
	vm_unmap_ram(pvVirtAddr, ui64Size >> OSGetPageShift());
	#endif
	}

	/*!
	******************************************************************************
	@Function SysDmaAllocMem

	@Description Allocates physically contiguous memory

	@Return PVRSRV_ERROR PVRSRV_OK on success. Otherwise, a PVRSRV_
	error code
	******************************************************************************/
	PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc)
	{
	PVRSRV_ERROR eError = PVRSRV_OK;
	struct device *psDev;
	struct page *psPage;
	size_t uiSize;

	if (psDmaAlloc == NULL \|\|
	psDmaAlloc->hHandle \|\|
	psDmaAlloc->pvVirtAddr \|\|
	psDmaAlloc->ui64Size == 0 \|\|
	psDmaAlloc->sBusAddr.uiAddr \|\|
	psDmaAlloc->pvOSDevice == NULL)
	{
	PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
	return PVRSRV_ERROR_INVALID_PARAMS;
	}

	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
	psDev = (struct device *)psDmaAlloc->pvOSDevice;

	#if !(defined(CONFIG_L4) \|\| defined(PVR_LINUX_PHYSMEM_SUPPRESS_DMA_AC))
	psDmaAlloc->hHandle = dma_alloc_coherent(psDev, uiSize, (dma_addr_t *)&psDmaAlloc->sBusAddr.uiAddr, GFP_KERNEL);
	#endif
	if (psDmaAlloc->hHandle)
	{
	psDmaAlloc->pvVirtAddr = psDmaAlloc->hHandle;

	PVR_DPF((PVR_DBG_MESSAGE,
	"Allocated DMA buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
	psDmaAlloc->pvVirtAddr,
	psDmaAlloc->sBusAddr.uiAddr,
	uiSize));
	}
	else if ((psPage = alloc_pages(GFP_KERNEL, get_order(uiSize))))
	{
	#if defined(CONFIG_L4)
	/* L4 is a para-virtualized environment, the PFN space is a virtual space and not physical space */
	psDmaAlloc->sBusAddr.uiAddr = l4x_virt_to_phys((void*)((unsigned long)page_to_pfn(psPage) << PAGE_SHIFT));
	#else
	psDmaAlloc->sBusAddr.uiAddr = dma_map_page(psDev, psPage, 0, uiSize, DMA_BIDIRECTIONAL);
	if (dma_mapping_error(psDev, psDmaAlloc->sBusAddr.uiAddr))
	{
	PVR_DPF((PVR_DBG_ERROR,
	"dma_map_page() failed, page 0x%p order %d",
	psPage,
	get_order(uiSize)));
	__free_pages(psPage, get_order(uiSize));
	goto e0;
	}
	psDmaAlloc->psPage = psPage;
	#endif

	psDmaAlloc->pvVirtAddr = SysDmaAcquireKernelAddress(psPage, uiSize, psDmaAlloc->pvOSDevice);
	if (! psDmaAlloc->pvVirtAddr)
	{
	PVR_DPF((PVR_DBG_ERROR,
	"SysDmaAcquireKernelAddress() failed, page 0x%p order %d",
	psPage,
	get_order(uiSize)));
	#if !defined(CONFIG_L4)
	dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
	#endif
	__free_pages(psPage, get_order(uiSize));
	goto e0;
	}

	PVR_DPF((PVR_DBG_MESSAGE,
	"Allocated contiguous buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
	psDmaAlloc->pvVirtAddr,
	psDmaAlloc->sBusAddr.uiAddr,
	uiSize));
	}
	else
	{
	PVR_DPF((PVR_DBG_ERROR, "Unable to allocate contiguous buffer, size: 0x"IMG_SIZE_FMTSPECX, uiSize));
	eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
	}

	e0:
	PVR_LOGR_IF_FALSE((psDmaAlloc->pvVirtAddr), "DMA/CMA allocation failed", PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES);
	return eError;
	}

	/*!
	******************************************************************************
	@Function SysDmaFreeMem

	@Description Free physically contiguous memory

	@Return void
	******************************************************************************/
	void SysDmaFreeMem(DMA_ALLOC *psDmaAlloc)
	{
	size_t uiSize;
	struct device *psDev;

	if (psDmaAlloc == NULL \|\|
	psDmaAlloc->ui64Size == 0 \|\|
	psDmaAlloc->pvOSDevice == NULL \|\|
	psDmaAlloc->pvVirtAddr == NULL \|\|
	psDmaAlloc->sBusAddr.uiAddr == 0)
	{
	PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
	return;
	}

	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
	psDev = (struct device *)psDmaAlloc->pvOSDevice;

	if (psDmaAlloc->pvVirtAddr != psDmaAlloc->hHandle)
	{
	SysDmaReleaseKernelAddress(psDmaAlloc->pvVirtAddr, uiSize);
	}

	if (! psDmaAlloc->hHandle)
	{
	struct page *psPage;
	#if defined(CONFIG_L4)
	psPage = pfn_to_page((unsigned long)l4x_phys_to_virt(psDmaAlloc->sBusAddr.uiAddr) >> PAGE_SHIFT);
	#else
	dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
	psPage = psDmaAlloc->psPage;
	#endif
	__free_pages(psPage, get_order(uiSize));
	return;
	}

	dma_free_coherent(psDev, uiSize, psDmaAlloc->hHandle, (dma_addr_t )psDmaAlloc->sBusAddr.uiAddr);
	}

	/*!
	******************************************************************************
	@Function SysDmaRegisterForIoRemapping

	@Description Registers DMA_ALLOC for manual I/O remapping

	@Return PVRSRV_ERROR PVRSRV_OK on success. Otherwise, a PVRSRV_
	error code
	******************************************************************************/
	PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
	{
	size_t uiSize;
	IMG_UINT32 ui32Idx;
	IMG_BOOL bTabEntryFound = IMG_TRUE;
	PVRSRV_ERROR eError = PVRSRV_ERROR_TOO_FEW_BUFFERS;

	if (psDmaAlloc == NULL \|\|
	psDmaAlloc->ui64Size == 0 \|\|
	psDmaAlloc->pvOSDevice == NULL \|\|
	psDmaAlloc->pvVirtAddr == NULL \|\|
	psDmaAlloc->sBusAddr.uiAddr == 0)
	{
	PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
	return PVRSRV_ERROR_INVALID_PARAMS;
	}

	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);

	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
	{
	/* Check if an I/O remap entry exists for remapping */
	if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == NULL)
	{
	PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == 0);
	PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].ui64Size == 0);
	break;
	}
	}

	if (ui32Idx >= DMA_MAX_IOREMAP_ENTRIES)
	{
	bTabEntryFound = IMG_FALSE;
	}

	if (bTabEntryFound)
	{
	IMG_BOOL bSameVAddr, bSamePAddr, bSameSize;

	bSamePAddr = gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == psDmaAlloc->sBusAddr.uiAddr;
	bSameVAddr = gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr;
	bSameSize = gsDmaIoRemapArray[ui32Idx].ui64Size == uiSize;

	if (bSameVAddr)
	{
	if (bSamePAddr && bSameSize)
	{
	eError = PVRSRV_OK;
	}
	else
	{
	eError = PVRSRV_ERROR_ALREADY_EXISTS;
	}
	}
	else
	{
	PVR_ASSERT(bSamePAddr == IMG_FALSE);

	gsDmaIoRemapArray[ui32Idx].ui64Size = uiSize;
	gsDmaIoRemapArray[ui32Idx].sBusAddr = psDmaAlloc->sBusAddr;
	gsDmaIoRemapArray[ui32Idx].pvVirtAddr = psDmaAlloc->pvVirtAddr;

	PVR_DPF((PVR_DBG_MESSAGE,
	"DMA: register I/O remap: "
	"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
	psDmaAlloc->pvVirtAddr,
	psDmaAlloc->sBusAddr.uiAddr,
	uiSize));

	gbEnableDmaIoRemapping = IMG_TRUE;
	eError = PVRSRV_OK;
	}
	}

	return eError;
	}

	/*!
	******************************************************************************
	@Function SysDmaDeregisterForIoRemapping

	@Description Deregisters DMA_ALLOC from manual I/O remapping

	@Return void
	******************************************************************************/
	void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
	{
	size_t uiSize;
	IMG_UINT32 ui32Idx;

	if (psDmaAlloc == NULL \|\|
	psDmaAlloc->ui64Size == 0 \|\|
	psDmaAlloc->pvOSDevice == NULL \|\|
	psDmaAlloc->pvVirtAddr == NULL \|\|
	psDmaAlloc->sBusAddr.uiAddr == 0)
	{
	PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
	return;
	}

	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);

	/* Remove specified entries from list of I/O remap entries */
	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
	{
	if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr)
	{
	gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr = 0;
	gsDmaIoRemapArray[ui32Idx].pvVirtAddr = NULL;
	gsDmaIoRemapArray[ui32Idx].ui64Size = 0;

	PVR_DPF((PVR_DBG_MESSAGE,
	"DMA: deregister I/O remap: "
	"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
	psDmaAlloc->pvVirtAddr,
	psDmaAlloc->sBusAddr.uiAddr,
	uiSize));

	break;
	}
	}

	/* Check if no other I/O remap entries exists for remapping */
	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
	{
	if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr != NULL)
	{
	break;
	}
	}

	if (ui32Idx == DMA_MAX_IOREMAP_ENTRIES)
	{
	/* No entries found so disable remapping */
	gbEnableDmaIoRemapping = IMG_FALSE;
	}
	}

	/*!
	******************************************************************************
	@Function SysDmaDevPAddrToCpuVAddr

	@Description Maps a DMA_ALLOC physical address to CPU virtual address

	@Return IMG_CPU_VIRTADDR on success. Otherwise, a NULL
	******************************************************************************/
	IMG_CPU_VIRTADDR SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size)
	{
	IMG_CPU_VIRTADDR pvDMAVirtAddr = NULL;
	DMA_ALLOC *psHeapDmaAlloc;
	IMG_UINT32 ui32Idx;

	if (gbEnableDmaIoRemapping == IMG_FALSE)
	{
	return pvDMAVirtAddr;
	}

	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
	{
	psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
	if (psHeapDmaAlloc->sBusAddr.uiAddr && uiAddr >= psHeapDmaAlloc->sBusAddr.uiAddr)
	{
	IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
	IMG_UINT64 uiOffset = uiAddr - psHeapDmaAlloc->sBusAddr.uiAddr;

	if (uiOffset < uiSpan)
	{
	PVR_ASSERT((uiOffset+ui64Size-1) < uiSpan);
	pvDMAVirtAddr = psHeapDmaAlloc->pvVirtAddr + uiOffset;

	PVR_DPF((PVR_DBG_MESSAGE,
	"DMA: remap: PA: 0x%llx => VA: 0x%p",
	uiAddr, pvDMAVirtAddr));

	break;
	}
	}
	}

	return pvDMAVirtAddr;
	}

	/*!
	******************************************************************************
	@Function SysDmaCpuVAddrToDevPAddr

	@Description Maps a DMA_ALLOC CPU virtual address to physical address

	@Return Non-zero value on success. Otherwise, a 0
	******************************************************************************/
	IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr)
	{
	IMG_UINT64 uiAddr = 0;
	DMA_ALLOC *psHeapDmaAlloc;
	IMG_UINT32 ui32Idx;

	if (gbEnableDmaIoRemapping == IMG_FALSE)
	{
	return uiAddr;
	}

	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
	{
	psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
	if (psHeapDmaAlloc->pvVirtAddr && pvDMAVirtAddr >= psHeapDmaAlloc->pvVirtAddr)
	{
	IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
	IMG_UINT64 uiOffset = pvDMAVirtAddr - psHeapDmaAlloc->pvVirtAddr;

	if (uiOffset < uiSpan)
	{
	uiAddr = psHeapDmaAlloc->sBusAddr.uiAddr + uiOffset;

	PVR_DPF((PVR_DBG_MESSAGE,
	"DMA: remap: VA: 0x%p => PA: 0x%llx",
	pvDMAVirtAddr, uiAddr));

	break;
	}
	}
	}

	return uiAddr;
	}

	/******************************************************************************
	End of file (dma_support.c)
	******************************************************************************/