/*************************************************************************/ /*!
@File
@Title          Linux OS PMR functions
@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
@License        Dual MIT/GPLv2

The contents of this file are subject to the MIT license as set out below.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 2 ("GPL") in which case the provisions
of GPL are applicable instead of those above.

If you wish to allow use of your version of this file only under the terms of
GPL, and not to allow others to use your version of this file under the terms
of the MIT license, indicate your decision by deleting the provisions above
and replace them with the notice and other provisions required by GPL as set
out in the file called "GPL-COPYING" included in this distribution. If you do
not delete the provisions above, a recipient may use your version of this file
under the terms of either the MIT license or GPL.

This License is also included in this distribution in the file called
"MIT-COPYING".

EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ /**************************************************************************/

#include <asm/page.h>
#include <linux/io.h>
#include <linux/mm.h>
#include <linux/dma-mapping.h>
#if defined(CONFIG_L4)
#include <asm/api-l4env/api.h>
#endif
#include <linux/version.h>
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
#include <linux/pfn_t.h>
#include <linux/pfn.h>
#endif

#include "img_defs.h"
#include "pvr_debug.h"
#include "allocmem.h"
#include "devicemem_server_utils.h"
#include "pmr.h"
#include "pmr_os.h"

#if defined(PVRSRV_ENABLE_PROCESS_STATS)
#include "process_stats.h"
#endif

#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
#include "mmap_stats.h"
#endif

#include "kernel_compatibility.h"

/*
 * x86_32:
 * Use vm_insert_page because remap_pfn_range has issues when mapping HIGHMEM
 * pages with default memory attributes; these HIGHMEM pages are skipped in
 * set_pages_array_[uc,wc] during allocation; see reserve_pfn_range().
 * Also vm_insert_page is faster.
 *
 * x86_64:
 * Use vm_insert_page because it is faster.
 *
 * Other platforms:
 * Use remap_pfn_range by default because it does not issue a cache flush.
 * It is known that ARM32 benefits from this. When other platforms become
 * available it has to be investigated if this assumption holds for them as well.
 *
 * Since vm_insert_page does more precise memory accounting we have the build
 * flag PVR_MMAP_USE_VM_INSERT that forces its use. This is useful as a debug
 * feature.
 *
 */
#if defined(CONFIG_X86) || defined(PVR_MMAP_USE_VM_INSERT)
#define PMR_OS_USE_VM_INSERT_PAGE 1
#endif

static void MMapPMROpen(struct vm_area_struct *ps_vma)
{
	PMR *psPMR = ps_vma->vm_private_data;

	/* Our VM flags should ensure this function never gets called */
	PVR_DPF((PVR_DBG_WARNING,
			 "%s: Unexpected mmap open call, this is probably an application bug.",
			 __func__));
	PVR_DPF((PVR_DBG_WARNING,
			 "%s: vma struct: 0x%p, vAddr: %#lX, length: %#lX, PMR pointer: 0x%p",
			 __func__,
			 ps_vma,
			 ps_vma->vm_start,
			 ps_vma->vm_end - ps_vma->vm_start,
			 psPMR));

	/* In case we get called anyway let's do things right by increasing the refcount and
	 * locking down the physical addresses. */
	PMRRefPMR(psPMR);

	if (PMRLockSysPhysAddresses(psPMR) != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "%s: Could not lock down physical addresses, aborting.", __func__));
		PMRUnrefPMR(psPMR);
	}
}

static void MMapPMRClose(struct vm_area_struct *ps_vma)
{
	PMR *psPMR = ps_vma->vm_private_data;

#if defined(PVRSRV_ENABLE_PROCESS_STATS)
#if	defined(PVRSRV_ENABLE_MEMORY_STATS)
	{
		uintptr_t vAddr = ps_vma->vm_start;

		while (vAddr < ps_vma->vm_end)
		{
			/* USER MAPPING */
			PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
			                                (IMG_UINT64)vAddr,
			                                OSGetCurrentClientProcessIDKM());
			vAddr += PAGE_SIZE;
		}
	}
#else
	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
	                            ps_vma->vm_end - ps_vma->vm_start,
	                            OSGetCurrentClientProcessIDKM());
#endif
#endif

	PMRUnlockSysPhysAddresses(psPMR);
	PMRUnrefPMR(psPMR);
}

/*
 * This vma operation is used to read data from mmap regions. It is called
 * by access_process_vm, which is called to handle PTRACE_PEEKDATA ptrace
 * requests and reads from /proc/<pid>/mem.
 */
static int MMapVAccess(struct vm_area_struct *ps_vma, unsigned long addr,
		       void *buf, int len, int write)
{
	PMR *psPMR = ps_vma->vm_private_data;
	unsigned long ulOffset = addr - ps_vma->vm_start;
	size_t uiBytesCopied;
	PVRSRV_ERROR eError;
	int iRetVal = -EINVAL;

	if (write)
	{
		eError = PMR_WriteBytes(psPMR,
					(IMG_DEVMEM_OFFSET_T) ulOffset,
					buf,
					len,
					&uiBytesCopied);
	}
	else
	{
		eError = PMR_ReadBytes(psPMR,
				       (IMG_DEVMEM_OFFSET_T) ulOffset,
				       buf,
				       len,
				       &uiBytesCopied);
	}

	if (eError != PVRSRV_OK)
	{
		PVR_DPF((PVR_DBG_ERROR, "%s: Error from %s (%d)",
			 __func__,
			 write ? "PMR_WriteBytes" : "PMR_ReadBytes",
			 eError));
	}
	else
	{
		iRetVal = uiBytesCopied;
	}

	return iRetVal;
}

static const struct vm_operations_struct gsMMapOps =
{
	.open = &MMapPMROpen,
	.close = &MMapPMRClose,
	.access = MMapVAccess,
};

static INLINE int _OSMMapPMR(PVRSRV_DEVICE_NODE *psDevNode,
							struct vm_area_struct *ps_vma,
							IMG_DEVMEM_OFFSET_T uiOffset,
							IMG_CPU_PHYADDR *psCpuPAddr,
							IMG_UINT32 uiLog2PageSize,
							IMG_BOOL bUseVMInsertPage,
							IMG_BOOL bUseMixedMap)
{
	IMG_INT32 iStatus;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
	pfn_t sPFN;
#else
	unsigned long uiPFN;
#endif

#if defined(CONFIG_L4)
	size_t size;
	IMG_CPU_VIRTADDR pvVAddr;
#if defined(ARM)
	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
#endif

	/* In L4 remaps from KM into UM is done via VA */
	pvVAddr = l4x_phys_to_virt(psCpuPAddr->uiAddr);
	if (pvVAddr == NULL)
	{
		return -1;
	}

	for (size = 0; size < 1ULL << uiLog2PageSize; size += PAGE_SIZE)
	{
		/* Fault-in pages now, ensure compiler does not optimise this out */
		*((volatile int*)pvVAddr + size) = *((volatile int*)pvVAddr + size);
	}

#if defined(ARM)
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
	sPFN = pfn_to_pfn_t(dma_to_pfn(dev, psCpuPAddr->uiAddr));
#else
	uiPFN = dma_to_pfn(dev, psCpuPAddr->uiAddr);
#endif
#else /* defined(ARM) */
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
	sPFN =  pfn_to_pfn_t(((uintptr_t) pvVAddr) >> PAGE_SHIFT);
#else
	uiPFN = ((uintptr_t) pvVAddr) >> PAGE_SHIFT;
	PVR_ASSERT(((IMG_UINT64)uiPFN << PAGE_SHIFT) == (IMG_UINT64)(uintptr_t)pvVAddr);
#endif
#endif
	PVR_ASSERT(bUseVMInsertPage == IMG_FALSE);
#else /* defined(CONFIG_L4) */
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
	sPFN = phys_to_pfn_t(psCpuPAddr->uiAddr, 0);
#else
	uiPFN = psCpuPAddr->uiAddr >> PAGE_SHIFT;
	PVR_ASSERT(((IMG_UINT64)uiPFN << PAGE_SHIFT) == psCpuPAddr->uiAddr);
#endif
#endif

	/*
	 * vm_insert_page() allows insertion of individual pages into user
	 * VMA space _only_ if page is a order-zero allocated page
	 */
	if (bUseVMInsertPage)
	{
		if (bUseMixedMap)
		{
			/*
			 * This path is just for debugging. It should be
			 * equivalent to the remap_pfn_range() path.
			 */
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0))
			vm_fault_t vmf;

			vmf = vmf_insert_mixed(ps_vma,
								  	ps_vma->vm_start + uiOffset,
								  	sPFN);
			if (vmf & VM_FAULT_ERROR)
			{
				iStatus = vm_fault_to_errno(vmf, 0);
			}
			else
			{
				iStatus = 0;
			}
#else
			iStatus = vm_insert_mixed(ps_vma,
									  ps_vma->vm_start + uiOffset,
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
									  sPFN);
#else
									  uiPFN);
#endif
#endif
		}
		else
		{
			/* Since kernel 3.7 this sets VM_MIXEDMAP internally */
			iStatus = vm_insert_page(ps_vma,
									 ps_vma->vm_start + uiOffset,
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
									 pfn_t_to_page(sPFN));
#else
									 pfn_to_page(uiPFN));
#endif
		}
	}
	else
	{
		/*
		   NOTE: Regarding absence of dma_mmap_coherent() in _OSMMapPMR()

		   The current services mmap model maps in a PMR's full-length size
		   into the user VMA & applies any user specified offset to the kernel
		   returned zero-offset based VA in services client; this essentially
		   means services server ignores ps_vma->vm_pgoff (this houses hPMR)
		   during a mmap call.

		   Furthermore, during a DMA/CMA memory allocation, multiple order-n
		   pages are used to satisfy an allocation request due to DMA/CMA
		   framework rounding-up allocation size to next power-of-two which
		   can lead to wasted memory (so we don't allocate using single call).

		   The combination of the above two issues mean that we cannot use the
		   dma_mmap_coherent() for a number of reasons outlined below:

		     - Services mmap semantics does not fit with dma_mmap_coherent()
		       which requires proper ps_vma->vm_pgoff; seeing this houses a
		       hPMR handle value, calls into dma_mmap_coherent() fails. This
		       could be avoided by forcing ps_vma->vm_pgoff to zero but the
		       ps_vma->vm_pgoff is applied to DMA bus address PFN and not
		       user VMA which is always mapped at ps_vma->vm_start.

		     - As multiple order-n pages are used for DMA/CMA allocations, a
		       single dma_mmap_coherent() call with a vma->vm_pgoff set to
		       zero cannot (maybe) be used because there is no guarantee that
		       all of the multiple order-n pages in the PMR are physically
		       contiguous from the first entry to the last. Whilst this is
		       highly likely to be the case, there is no guarantee that it
		       will be so we cannot depend on this being the case.

		   The solution is to manually mmap DMA/CMA pages into user VMA
		   using remap_pfn_range() directly. Furthermore, accounting is
		   always compromised for DMA/CMA allocations.
		*/
		size_t uiNumContiguousBytes = 1ULL << uiLog2PageSize;

		iStatus = remap_pfn_range(ps_vma,
								  ps_vma->vm_start + uiOffset,
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
								  pfn_t_to_pfn(sPFN),
#else
								  uiPFN,
#endif
								  uiNumContiguousBytes,
								  ps_vma->vm_page_prot);
	}

	return iStatus;
}

PVRSRV_ERROR
OSMMapPMRGeneric(PMR *psPMR, PMR_MMAP_DATA pOSMMapData)
{
	struct vm_area_struct *ps_vma = pOSMMapData;
	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
	PVRSRV_ERROR eError;
	size_t uiLength;
	IMG_INT32 iStatus;
	IMG_DEVMEM_OFFSET_T uiOffset;
	IMG_UINT32 ui32CPUCacheFlags;
	pgprot_t sPageProt;
	IMG_CPU_PHYADDR asCpuPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
	IMG_UINT32 uiOffsetIdx;
	IMG_UINT32 uiNumOfPFNs;
	IMG_UINT32 uiLog2PageSize;
	IMG_CPU_PHYADDR *psCpuPAddr;
	IMG_BOOL *pbValid;
	IMG_BOOL bUseMixedMap = IMG_FALSE;
	IMG_BOOL bUseVMInsertPage = IMG_FALSE;

	eError = PMRLockSysPhysAddresses(psPMR);
	if (eError != PVRSRV_OK)
	{
		goto e0;
	}

	if (((ps_vma->vm_flags & VM_WRITE) != 0) &&
		((ps_vma->vm_flags & VM_SHARED) == 0))
	{
		eError = PVRSRV_ERROR_INVALID_PARAMS;
		goto e1;
	}

	sPageProt = vm_get_page_prot(ps_vma->vm_flags);

	eError = DevmemCPUCacheMode(psDevNode,
	                            PMR_Flags(psPMR),
	                            &ui32CPUCacheFlags);
	if (eError != PVRSRV_OK)
	{
		goto e0;
	}

	switch (ui32CPUCacheFlags)
	{
		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
				sPageProt = pgprot_noncached(sPageProt);
				break;

		case PVRSRV_MEMALLOCFLAG_CPU_WRITE_COMBINE:
				sPageProt = pgprot_writecombine(sPageProt);
				break;

		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
		{
/* Do not set to write-combine for plato */
#if !defined(PLATO_MEMORY_CONFIG)
				PHYS_HEAP *psPhysHeap = PMR_PhysHeap(psPMR);

				if (PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_LMA)
					sPageProt = pgprot_writecombine(sPageProt);
#endif
				break;
		}

		default:
				eError = PVRSRV_ERROR_INVALID_PARAMS;
				goto e1;
	}
	ps_vma->vm_page_prot = sPageProt;

	ps_vma->vm_flags |= VM_IO;

	/* Don't include the mapping in core dumps */
	ps_vma->vm_flags |= VM_DONTDUMP;

	/*
	 * Disable mremap because our nopage handler assumes all
	 * page requests have already been validated.
	 */
	ps_vma->vm_flags |= VM_DONTEXPAND;

	/* Don't allow mapping to be inherited across a process fork */
	ps_vma->vm_flags |= VM_DONTCOPY;

	uiLength = ps_vma->vm_end - ps_vma->vm_start;

	/* Is this mmap targeting non order-zero pages or does it use pfn mappings?
	 * If yes, don't use vm_insert_page */
	uiLog2PageSize = PMR_GetLog2Contiguity(psPMR);
#if defined(PMR_OS_USE_VM_INSERT_PAGE)
	bUseVMInsertPage = (uiLog2PageSize == PAGE_SHIFT) && (PMR_GetType(psPMR) != PMR_TYPE_EXTMEM);
#if defined(CONFIG_L4)
	/* L4 uses CMA allocations */
	bUseVMInsertPage = IMG_FALSE;
#endif
#endif

	/* Can we use stack allocations */
	uiNumOfPFNs = uiLength >> uiLog2PageSize;
	if (uiNumOfPFNs > PMR_MAX_TRANSLATION_STACK_ALLOC)
	{
		psCpuPAddr = OSAllocMem(uiNumOfPFNs * sizeof(*psCpuPAddr));
		if (psCpuPAddr == NULL)
		{
			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
			goto e1;
		}

		/* Should allocation fail, clean-up here before exiting */
		pbValid = OSAllocMem(uiNumOfPFNs * sizeof(*pbValid));
		if (pbValid == NULL)
		{
			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
			OSFreeMem(psCpuPAddr);
			goto e2;
		}
	}
	else
	{
		psCpuPAddr = asCpuPAddr;
		pbValid = abValid;
	}

	/* Obtain map range pfns */
	eError = PMR_CpuPhysAddr(psPMR,
				 uiLog2PageSize,
				 uiNumOfPFNs,
				 0,
				 psCpuPAddr,
				 pbValid);
	if (eError != PVRSRV_OK)
	{
		goto e3;
	}

	/*
	 * Scan the map range for pfns without struct page* handling. If
	 * we find one, this is a mixed map, and we can't use vm_insert_page()
	 * NOTE: vm_insert_page() allows insertion of individual pages into user
	 * VMA space _only_ if said page is an order-zero allocated page.
	 */
	if (bUseVMInsertPage)
	{
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
		pfn_t sPFN;
#else
		unsigned long uiPFN;
#endif

		for (uiOffsetIdx = 0; uiOffsetIdx < uiNumOfPFNs; ++uiOffsetIdx)
		{
			if (pbValid[uiOffsetIdx])
			{
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
				sPFN = phys_to_pfn_t(psCpuPAddr[uiOffsetIdx].uiAddr, 0);

				if (!pfn_t_valid(sPFN) || page_count(pfn_t_to_page(sPFN)) == 0)
#else
				uiPFN = psCpuPAddr[uiOffsetIdx].uiAddr >> PAGE_SHIFT;
				PVR_ASSERT(((IMG_UINT64)uiPFN << PAGE_SHIFT) == psCpuPAddr[uiOffsetIdx].uiAddr);

				if (!pfn_valid(uiPFN) || page_count(pfn_to_page(uiPFN)) == 0)
#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)) */
				{
					bUseMixedMap = IMG_TRUE;
					break;
				}
			}
		}

		if (bUseMixedMap)
		{
			ps_vma->vm_flags |= VM_MIXEDMAP;
		}
	}
	else
	{
		ps_vma->vm_flags |= VM_PFNMAP;
	}

	/* For each PMR page-size contiguous bytes, map page(s) into user VMA */
	for (uiOffset = 0; uiOffset < uiLength; uiOffset += 1ULL<<uiLog2PageSize)
	{
		uiOffsetIdx = uiOffset >> uiLog2PageSize;
		/*
		 * Only map in pages that are valid, any that aren't will be
		 * picked up by the nopage handler which will return a zeroed
		 * page for us.
		 */
		if (pbValid[uiOffsetIdx])
		{
			iStatus = _OSMMapPMR(psDevNode,
								 ps_vma,
								 uiOffset,
								 &psCpuPAddr[uiOffsetIdx],
								 uiLog2PageSize,
								 bUseVMInsertPage,
								 bUseMixedMap);
			if (iStatus)
			{
				/* Failure error code doesn't get propagated */
				eError = PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
				PVR_ASSERT(0);
				goto e3;
			}
		}
#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
#define PMR_OS_BAD_CPUADDR 0x0BAD0BAD
		{
			IMG_CPU_PHYADDR sPAddr;
			sPAddr.uiAddr = pbValid[uiOffsetIdx] ?
					psCpuPAddr[uiOffsetIdx].uiAddr :
					IMG_CAST_TO_CPUPHYADDR_UINT(PMR_OS_BAD_CPUADDR);

			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
										(void*)(uintptr_t)(ps_vma->vm_start + uiOffset),
										sPAddr,
										1<<uiLog2PageSize,
										NULL,
										OSGetCurrentClientProcessIDKM());
		}
#undef PMR_OS_BAD_CPUADDR
#endif
	}

#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES, uiNumOfPFNs * PAGE_SIZE, OSGetCurrentClientProcessIDKM());
#endif

	if (psCpuPAddr != asCpuPAddr)
	{
		OSFreeMem(psCpuPAddr);
		OSFreeMem(pbValid);
	}

	/* let us see the PMR so we can unlock it later */
	ps_vma->vm_private_data = psPMR;

	/* Install open and close handlers for ref-counting */
	ps_vma->vm_ops = &gsMMapOps;

	/*
	 * Take a reference on the PMR so that it can't be freed while mapped
	 * into the user process.
	 */
	PMRRefPMR(psPMR);

#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
	/* record the stats */
	MMapStatsAddOrUpdatePMR(psPMR, uiLength);
#endif

	return PVRSRV_OK;

	/* Error exit paths follow */
 e3:
	if (pbValid != abValid)
	{
		OSFreeMem(pbValid);
	}
 e2:
	if (psCpuPAddr != asCpuPAddr)
	{
		OSFreeMem(psCpuPAddr);
	}
 e1:
	PMRUnlockSysPhysAddresses(psPMR);
 e0:
	return eError;
}