| /*************************************************************************/ /*! |
| @File |
| @Title Device Memory Management |
| @Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved |
| @Description Front End (nominally Client side part, but now invokable |
| from server too) of device memory management |
| @License Dual MIT/GPLv2 |
| |
| The contents of this file are subject to the MIT license as set out below. |
| |
| Permission is hereby granted, free of charge, to any person obtaining a copy |
| of this software and associated documentation files (the "Software"), to deal |
| in the Software without restriction, including without limitation the rights |
| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| copies of the Software, and to permit persons to whom the Software is |
| furnished to do so, subject to the following conditions: |
| |
| The above copyright notice and this permission notice shall be included in |
| all copies or substantial portions of the Software. |
| |
| Alternatively, the contents of this file may be used under the terms of |
| the GNU General Public License Version 2 ("GPL") in which case the provisions |
| of GPL are applicable instead of those above. |
| |
| If you wish to allow use of your version of this file only under the terms of |
| GPL, and not to allow others to use your version of this file under the terms |
| of the MIT license, indicate your decision by deleting the provisions above |
| and replace them with the notice and other provisions required by GPL as set |
| out in the file called "GPL-COPYING" included in this distribution. If you do |
| not delete the provisions above, a recipient may use your version of this file |
| under the terms of either the MIT license or GPL. |
| |
| This License is also included in this distribution in the file called |
| "MIT-COPYING". |
| |
| EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS |
| PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING |
| BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR |
| PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR |
| COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER |
| IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| */ /**************************************************************************/ |
| |
| #include "devicemem.h" |
| #include "img_types.h" |
| #include "img_defs.h" |
| #include "pvr_debug.h" |
| #include "pvrsrv_error.h" |
| #include "allocmem.h" |
| #include "ra.h" |
| #include "osfunc.h" |
| #include "osmmap.h" |
| #include "devicemem_utils.h" |
| #include "client_mm_bridge.h" |
| #include "client_cache_bridge.h" |
| #include "services_km.h" |
| |
| #if defined(PDUMP) |
| #if defined(__KERNEL__) |
| #include "pdump_km.h" |
| #else |
| #include "pdump_um.h" |
| #endif |
| #include "devicemem_pdump.h" |
| #endif |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| #include "client_ri_bridge.h" |
| #endif |
| #include "client_devicememhistory_bridge.h" |
| #include "info_page_client.h" |
| |
| #include "rgx_heaps.h" |
| #if defined(__KERNEL__) |
| #include "pvrsrv.h" |
| #include "rgxdefs_km.h" |
| #include "rgx_bvnc_defs_km.h" |
| #include "device.h" |
| #include "rgxdevice.h" |
| #include "pvr_ricommon.h" |
| #if defined(LINUX) |
| #include "linux/kernel.h" |
| #endif |
| #else |
| #include "rgxdefs.h" |
| #endif |
| |
| #if defined(__KERNEL__) && defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| extern PVRSRV_ERROR RIDumpAllKM(void); |
| #endif |
| |
| #if defined(__KERNEL__) |
| #define GET_ERROR_STRING(eError) PVRSRVGetErrorString(eError) |
| #else |
| #define GET_ERROR_STRING(eError) PVRSRVGetErrorString(eError) |
| #endif |
| |
| #if defined(__KERNEL__) |
| /* Derive the virtual from the hPMR */ |
| static |
| IMG_UINT64 _GuestFWHeapVA(PMR *psPMR, PVRSRV_DEVICE_NODE *psDevNode) |
| { |
| PVRSRV_ERROR eError; |
| IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS; |
| |
| IMG_DEV_PHYADDR sDevAddr; |
| IMG_BOOL bValid; |
| |
| PHYS_HEAP *psPhysHeap = psDevNode->apsPhysHeap[PVRSRV_DEVICE_PHYS_HEAP_FW_LOCAL]; |
| IMG_DEV_PHYADDR sHeapAddr; |
| |
| eError = PhysHeapRegionGetDevPAddr(psPhysHeap, 0, &sHeapAddr); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_LOG_IF_ERROR(eError, "_GuestFWHeapVA: PMRLockSysPhysAddr"); |
| goto fail; |
| } |
| |
| #if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES) |
| { |
| IMG_DEV_PHYADDR sDevPAddrCorrected; |
| |
| PhysHeapCpuPAddrToDevPAddr(psPhysHeap,1,&sDevPAddrCorrected,(IMG_CPU_PHYADDR *)&sHeapAddr); |
| sHeapAddr.uiAddr = sDevPAddrCorrected.uiAddr; |
| } |
| #endif |
| |
| eError = PMRLockSysPhysAddresses(psPMR); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_LOG_IF_ERROR(eError, "_GuestFWHeapVA: PMRLockSysPhysAddr"); |
| goto fail; |
| } |
| |
| eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), 1, 0, &sDevAddr, &bValid); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_LOG_IF_ERROR(eError, "_GuestFWHeapVA: PMR_DevPhysAddr"); |
| eError = PMRUnlockSysPhysAddresses(psPMR); |
| PVR_LOG_IF_ERROR(eError, "_GuestFWHeapVA: PMRUnlockSysPhysAddr"); |
| goto fail; |
| } |
| |
| eError = PMRUnlockSysPhysAddresses(psPMR); |
| PVR_LOG_IF_ERROR(eError, "_GuestFWHeapVA: PMRUnlockSysPhysAddr"); |
| |
| ui64OptionalMapAddress = RGX_FIRMWARE_RAW_HEAP_BASE | (sDevAddr.uiAddr - sHeapAddr.uiAddr); |
| |
| PVR_DPF((PVR_DBG_ALLOC, "%s: RGX_FIRMWARE_RAW_HEAP_BASE = 0x%"IMG_UINT64_FMTSPECx" sDevAddr.uiAddr = 0x%"IMG_UINT64_FMTSPECx" sHeapAddr.uiAddr = 0x%"IMG_UINT64_FMTSPECx" => ui64OptionalMapAddress = 0x%"IMG_UINT64_FMTSPECx, |
| __func__, (IMG_UINT64) RGX_FIRMWARE_RAW_HEAP_BASE, sDevAddr.uiAddr, sHeapAddr.uiAddr, ui64OptionalMapAddress)); |
| fail: |
| return ui64OptionalMapAddress; |
| } |
| #endif |
| |
| /***************************************************************************** |
| * Sub allocation internals * |
| *****************************************************************************/ |
| |
| static INLINE void |
| _CheckAnnotationLength(const IMG_CHAR *pszAnnotation) |
| { |
| IMG_UINT32 length = OSStringLength(pszAnnotation); |
| |
| if (length >= DEVMEM_ANNOTATION_MAX_LEN) |
| { |
| PVR_DPF((PVR_DBG_WARNING, "%s: Annotation \"%s\" has been truncated to %d characters from %d characters", |
| __func__, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN - 1, length)); |
| } |
| } |
| |
| static PVRSRV_ERROR |
| _AllocateDeviceMemory(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_UINT32 uiLog2Quantum, |
| IMG_DEVMEM_SIZE_T uiSize, |
| IMG_DEVMEM_SIZE_T uiChunkSize, |
| IMG_UINT32 ui32NumPhysChunks, |
| IMG_UINT32 ui32NumVirtChunks, |
| IMG_UINT32 *pui32MappingTable, |
| IMG_DEVMEM_ALIGN_T uiAlign, |
| DEVMEM_FLAGS_T uiFlags, |
| IMG_BOOL bExportable, |
| const IMG_CHAR *pszAnnotation, |
| DEVMEM_IMPORT **ppsImport) |
| { |
| DEVMEM_IMPORT *psImport; |
| DEVMEM_FLAGS_T uiPMRFlags; |
| IMG_HANDLE hPMR; |
| PVRSRV_ERROR eError; |
| |
| eError = _DevmemImportStructAlloc(hDevConnection, |
| &psImport); |
| if (eError != PVRSRV_OK) |
| { |
| goto failAlloc; |
| } |
| |
| /* check if shift value is not too big (sizeof(1ULL)) */ |
| PVR_ASSERT(uiLog2Quantum < sizeof(unsigned long long) * 8); |
| /* Check the size is a multiple of the quantum */ |
| PVR_ASSERT((uiSize & ((1ULL<<uiLog2Quantum)-1)) == 0); |
| |
| _CheckAnnotationLength(pszAnnotation); |
| |
| /* Pass only the PMR flags down */ |
| uiPMRFlags = uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK; |
| eError = BridgePhysmemNewRamBackedPMR(GetBridgeHandle(hDevConnection), |
| uiSize, |
| uiChunkSize, |
| ui32NumPhysChunks, |
| ui32NumVirtChunks, |
| pui32MappingTable, |
| uiLog2Quantum, |
| uiPMRFlags, |
| OSStringNLength(pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN - 1) + 1, |
| pszAnnotation, |
| OSGetCurrentProcessID(), |
| &hPMR); |
| |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Failed to allocate memory for %s (%s)", |
| __func__, |
| pszAnnotation, |
| PVRSRVGETERRORSTRING(eError))); |
| goto failPMR; |
| } |
| |
| _DevmemImportStructInit(psImport, |
| uiSize, |
| uiAlign, |
| uiFlags, |
| hPMR, |
| bExportable ? DEVMEM_PROPERTIES_EXPORTABLE : 0); |
| |
| *ppsImport = psImport; |
| return PVRSRV_OK; |
| |
| failPMR: |
| _DevmemImportDiscard(psImport); |
| failAlloc: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| |
| return eError; |
| } |
| |
| |
| /***************************************************************************** |
| * Sub allocation internals * |
| *****************************************************************************/ |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DeviceMemChangeSparse(DEVMEM_MEMDESC *psMemDesc, |
| IMG_UINT32 ui32AllocPageCount, |
| IMG_UINT32 *paui32AllocPageIndices, |
| IMG_UINT32 ui32FreePageCount, |
| IMG_UINT32 *pauiFreePageIndices, |
| SPARSE_MEM_RESIZE_FLAGS uiSparseFlags) |
| { |
| PVRSRV_ERROR eError = PVRSRV_ERROR_INVALID_PARAMS; |
| DEVMEM_IMPORT *psImport = psMemDesc->psImport; |
| SHARED_DEV_CONNECTION hDevConnection; |
| IMG_HANDLE hPMR; |
| IMG_HANDLE hSrvDevMemHeap; |
| POS_LOCK hLock; |
| IMG_DEV_VIRTADDR sDevVAddr; |
| IMG_CPU_VIRTADDR pvCpuVAddr; |
| |
| if (NULL == psImport) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Sparse memory import", __func__)); |
| goto e0; |
| } |
| |
| hDevConnection = psImport->hDevConnection; |
| hPMR = psImport->hPMR; |
| hLock = psImport->hLock; |
| sDevVAddr = psImport->sDeviceImport.sDevVAddr; |
| pvCpuVAddr = psImport->sCPUImport.pvCPUVAddr; |
| |
| if (NULL == hDevConnection) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Bridge handle", __func__)); |
| goto e0; |
| } |
| |
| if (NULL == hPMR) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Invalid PMR handle", __func__)); |
| goto e0; |
| } |
| |
| if ((uiSparseFlags & SPARSE_RESIZE_BOTH) && (0 == sDevVAddr.uiAddr)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Device Virtual Map", __func__)); |
| goto e0; |
| } |
| |
| if ((uiSparseFlags & SPARSE_MAP_CPU_ADDR) && (NULL == pvCpuVAddr)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Invalid CPU Virtual Map", __func__)); |
| goto e0; |
| } |
| |
| if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_SECURE) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Secure buffers currently do not support sparse changes", |
| __func__)); |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto e0; |
| } |
| |
| if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: This memory descriptor doesn't support sparse changes", |
| __func__)); |
| eError = PVRSRV_ERROR_INVALID_REQUEST; |
| goto e0; |
| } |
| |
| hSrvDevMemHeap = psImport->sDeviceImport.psHeap->hDevMemServerHeap; |
| |
| OSLockAcquire(hLock); |
| |
| eError = BridgeChangeSparseMem(GetBridgeHandle(hDevConnection), |
| hSrvDevMemHeap, |
| hPMR, |
| ui32AllocPageCount, |
| paui32AllocPageIndices, |
| ui32FreePageCount, |
| pauiFreePageIndices, |
| uiSparseFlags, |
| psImport->uiFlags, |
| sDevVAddr, |
| (IMG_UINT64)((uintptr_t)pvCpuVAddr)); |
| |
| OSLockRelease(hLock); |
| |
| if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) |
| { |
| BridgeDevicememHistorySparseChange(GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->psImport->hPMR, |
| psMemDesc->uiOffset, |
| psMemDesc->sDeviceMemDesc.sDevVAddr, |
| psMemDesc->uiAllocSize, |
| psMemDesc->szText, |
| DevmemGetHeapLog2PageSize(psImport->sDeviceImport.psHeap), |
| ui32AllocPageCount, |
| paui32AllocPageIndices, |
| ui32FreePageCount, |
| pauiFreePageIndices, |
| psMemDesc->ui32AllocationIndex, |
| &psMemDesc->ui32AllocationIndex); |
| } |
| |
| #ifdef PVRSRV_UNMAP_ON_SPARSE_CHANGE |
| if ((PVRSRV_OK == eError) && (psMemDesc->sCPUMemDesc.ui32RefCount)) |
| { |
| /* |
| * Release the CPU Virtual mapping here |
| * the caller is supposed to map entire range again |
| */ |
| DevmemReleaseCpuVirtAddr(psMemDesc); |
| } |
| #endif |
| |
| e0: |
| return eError; |
| } |
| |
| static void |
| _FreeDeviceMemory(DEVMEM_IMPORT *psImport) |
| { |
| _DevmemImportStructRelease(psImport); |
| } |
| |
| static PVRSRV_ERROR |
| _SubAllocImportAlloc(RA_PERARENA_HANDLE hArena, |
| RA_LENGTH_T uiSize, |
| RA_FLAGS_T _flags, |
| const IMG_CHAR *pszAnnotation, |
| /* returned data */ |
| RA_BASE_T *puiBase, |
| RA_LENGTH_T *puiActualSize, |
| RA_PERISPAN_HANDLE *phImport) |
| { |
| /* When suballocations need a new lump of memory, the RA calls |
| back here. Later, in the kernel, we must construct a new PMR |
| and a pairing between the new lump of virtual memory and the |
| PMR (whether or not such PMR is backed by physical memory) */ |
| DEVMEM_HEAP *psHeap; |
| DEVMEM_IMPORT *psImport; |
| IMG_DEVMEM_ALIGN_T uiAlign; |
| PVRSRV_ERROR eError; |
| IMG_UINT32 ui32MappingTable = 0; |
| DEVMEM_FLAGS_T uiFlags = (DEVMEM_FLAGS_T) _flags; |
| IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS; |
| |
| /* Per-arena private handle is, for us, the heap */ |
| psHeap = hArena; |
| |
| /* align to the l.s.b. of the size... e.g. 96kiB aligned to |
| 32kiB. NB: There is an argument to say that the RA should never |
| ask us for Non-power-of-2 size anyway, but I don't want to make |
| that restriction arbitrarily now */ |
| uiAlign = uiSize & ~(uiSize-1); |
| |
| /* Technically this is only required for guest drivers due to |
| fw heaps being pre-allocated and pre-mapped resulting in |
| a 1:1 (i.e. virtual : physical) offset correlation but we |
| force this behaviour for all drivers to maintain consistency |
| (i.e. heap->VA uiAlign <= heap->PA uiLog2Quantum) */ |
| if (uiAlign > (IMG_DEVMEM_ALIGN_T)(1ULL << psHeap->uiLog2Quantum)) |
| { |
| uiAlign = (IMG_DEVMEM_ALIGN_T)(1ULL << psHeap->uiLog2Quantum); |
| } |
| |
| /* The RA should not have invoked us with a size that is not a |
| multiple of the quantum anyway */ |
| PVR_ASSERT((uiSize & ((1ULL<<psHeap->uiLog2Quantum)-1)) == 0); |
| |
| eError = _AllocateDeviceMemory(psHeap->psCtx->hDevConnection, |
| psHeap->uiLog2Quantum, |
| uiSize, |
| uiSize, |
| 1, |
| 1, |
| &ui32MappingTable, |
| uiAlign, |
| uiFlags, |
| IMG_FALSE, |
| "PMR sub-allocated", |
| &psImport); |
| if (eError != PVRSRV_OK) |
| { |
| goto failAlloc; |
| } |
| |
| #if defined(PDUMP) && defined(DEBUG) |
| #if defined(__KERNEL__) |
| PDUMPCOMMENTWITHFLAGS(PDUMP_CONT, |
| "Created PMR for sub-allocations with handle ID: 0x%p Annotation: \"%s\" (PID %u)", |
| psImport->hPMR, pszAnnotation, OSGetCurrentProcessID()); |
| #else |
| PDUMPCOMMENTF(psHeap->psCtx->hDevConnection, PDUMP_FLAGS_CONTINUOUS, |
| "Created PMR for sub-allocations with handle ID: %p Annotation: \"%s\" (PID %u)", |
| psImport->hPMR, pszAnnotation, OSGetCurrentProcessID()); |
| #endif |
| #else |
| PVR_UNREFERENCED_PARAMETER(pszAnnotation); |
| #endif |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI)) |
| { |
| #if defined(__KERNEL__) |
| PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection; |
| PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDevNode->pvDevice; |
| |
| PVR_ASSERT(PVRSRV_CHECK_FW_LOCAL(uiFlags)); |
| |
| /* If allocation is made by the Kernel from the firmware heap, account for it |
| * under the PVR_SYS_ALLOC_PID. |
| */ |
| if ((psHeap == psDevInfo->psFirmwareMainHeap) || (psHeap == psDevInfo->psFirmwareConfigHeap)) |
| { |
| eError = BridgeRIWritePMREntryWithOwner (GetBridgeHandle(psImport->hDevConnection), |
| psImport->hPMR, |
| PVR_SYS_ALLOC_PID); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntryWithOwner failed (Error=%d)", __func__, eError)); |
| } |
| } |
| else |
| #endif |
| { |
| eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection), |
| psImport->hPMR); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntry failed (Error=%d)", __func__, eError)); |
| } |
| } |
| } |
| #endif |
| |
| #if defined(__KERNEL__) |
| { |
| PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection; |
| PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDevNode->pvDevice; |
| |
| if (((psHeap == psDevInfo->psFirmwareMainHeap) || |
| (psHeap == psDevInfo->psFirmwareConfigHeap)) && |
| PVRSRV_VZ_MODE_IS(DRIVER_MODE_GUEST)) |
| { |
| ui64OptionalMapAddress = _GuestFWHeapVA(psImport->hPMR, psDevNode); |
| } |
| } |
| #endif |
| |
| /* |
| Suballocations always get mapped into the device was we need to |
| key the RA off something and as we can't export suballocations |
| there is no valid reason to request an allocation an not map it |
| */ |
| eError = _DevmemImportStructDevMap(psHeap, |
| IMG_TRUE, |
| psImport, |
| ui64OptionalMapAddress); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMap; |
| } |
| |
| /* Mark this import struct as zeroed so we can save some PDump LDBs |
| * and do not have to CPU map + memset()*/ |
| if (uiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) |
| { |
| psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_ZEROED; |
| } |
| else if (uiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC) |
| { |
| psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_POISONED; |
| } |
| psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_CLEAN; |
| |
| *puiBase = psImport->sDeviceImport.sDevVAddr.uiAddr; |
| *puiActualSize = uiSize; |
| *phImport = psImport; |
| |
| return PVRSRV_OK; |
| |
| /* error exit paths follow */ |
| |
| failMap: |
| _FreeDeviceMemory(psImport); |
| failAlloc: |
| |
| return eError; |
| } |
| |
| static void |
| _SubAllocImportFree(RA_PERARENA_HANDLE hArena, |
| RA_BASE_T uiBase, |
| RA_PERISPAN_HANDLE hImport) |
| { |
| DEVMEM_IMPORT *psImport = hImport; |
| |
| PVR_ASSERT(psImport != NULL); |
| PVR_ASSERT(hArena == psImport->sDeviceImport.psHeap); |
| PVR_ASSERT(uiBase == psImport->sDeviceImport.sDevVAddr.uiAddr); |
| |
| _DevmemImportStructDevUnmap(psImport); |
| _DevmemImportStructRelease(psImport); |
| } |
| |
| /***************************************************************************** |
| * Devmem context internals * |
| *****************************************************************************/ |
| |
| static PVRSRV_ERROR |
| _PopulateContextFromBlueprint(struct _DEVMEM_CONTEXT_ *psCtx, |
| DEVMEM_HEAPCFGID uiHeapBlueprintID) |
| { |
| PVRSRV_ERROR eError; |
| PVRSRV_ERROR eError2; |
| struct _DEVMEM_HEAP_ **ppsHeapArray; |
| IMG_UINT32 uiNumHeaps; |
| IMG_UINT32 uiHeapsToUnwindOnError; |
| IMG_UINT32 uiHeapIndex; |
| IMG_DEV_VIRTADDR sDevVAddrBase; |
| IMG_CHAR aszHeapName[DEVMEM_HEAPNAME_MAXLENGTH]; |
| IMG_DEVMEM_SIZE_T uiHeapLength; |
| IMG_DEVMEM_LOG2ALIGN_T uiLog2DataPageSize; |
| IMG_DEVMEM_LOG2ALIGN_T uiLog2ImportAlignment; |
| IMG_DEVMEM_LOG2ALIGN_T uiLog2TilingStrideFactor; |
| |
| eError = DevmemHeapCount(psCtx->hDevConnection, |
| uiHeapBlueprintID, |
| &uiNumHeaps); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| if (uiNumHeaps == 0) |
| { |
| ppsHeapArray = NULL; |
| } |
| else |
| { |
| ppsHeapArray = OSAllocMem(sizeof(*ppsHeapArray) * uiNumHeaps); |
| if (ppsHeapArray == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e0; |
| } |
| } |
| |
| uiHeapsToUnwindOnError = 0; |
| |
| for (uiHeapIndex = 0; uiHeapIndex < uiNumHeaps; uiHeapIndex++) |
| { |
| eError = DevmemHeapDetails(psCtx->hDevConnection, |
| uiHeapBlueprintID, |
| uiHeapIndex, |
| &aszHeapName[0], |
| sizeof(aszHeapName), |
| &sDevVAddrBase, |
| &uiHeapLength, |
| &uiLog2DataPageSize, |
| &uiLog2ImportAlignment, |
| &uiLog2TilingStrideFactor); |
| if (eError != PVRSRV_OK) |
| { |
| goto e1; |
| } |
| |
| eError = DevmemCreateHeap(psCtx, |
| sDevVAddrBase, |
| uiHeapLength, |
| uiLog2DataPageSize, |
| uiLog2ImportAlignment, |
| uiLog2TilingStrideFactor, |
| aszHeapName, |
| uiHeapBlueprintID, |
| &ppsHeapArray[uiHeapIndex]); |
| if (eError != PVRSRV_OK) |
| { |
| goto e1; |
| } |
| |
| uiHeapsToUnwindOnError = uiHeapIndex + 1; |
| } |
| |
| psCtx->uiAutoHeapCount = uiNumHeaps; |
| psCtx->ppsAutoHeapArray = ppsHeapArray; |
| |
| PVR_ASSERT(psCtx->uiNumHeaps >= psCtx->uiAutoHeapCount); |
| PVR_ASSERT(psCtx->uiAutoHeapCount == uiNumHeaps); |
| |
| return PVRSRV_OK; |
| |
| /* error exit paths */ |
| e1: |
| for (uiHeapIndex = 0; uiHeapIndex < uiHeapsToUnwindOnError; uiHeapIndex++) |
| { |
| eError2 = DevmemDestroyHeap(ppsHeapArray[uiHeapIndex]); |
| PVR_ASSERT(eError2 == PVRSRV_OK); |
| } |
| |
| if (uiNumHeaps != 0) |
| { |
| OSFreeMem(ppsHeapArray); |
| } |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| static PVRSRV_ERROR |
| _UnpopulateContextFromBlueprint(struct _DEVMEM_CONTEXT_ *psCtx) |
| { |
| PVRSRV_ERROR eReturn = PVRSRV_OK; |
| PVRSRV_ERROR eError2; |
| IMG_UINT32 uiHeapIndex; |
| IMG_BOOL bDoCheck = IMG_TRUE; |
| #if defined(__KERNEL__) |
| PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData(); |
| if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK) |
| { |
| bDoCheck = IMG_FALSE; |
| } |
| #endif |
| |
| for (uiHeapIndex = 0; uiHeapIndex < psCtx->uiAutoHeapCount; uiHeapIndex++) |
| { |
| if (!psCtx->ppsAutoHeapArray[uiHeapIndex]) |
| { |
| continue; |
| } |
| |
| eError2 = DevmemDestroyHeap(psCtx->ppsAutoHeapArray[uiHeapIndex]); |
| if (eError2 != PVRSRV_OK) |
| { |
| eReturn = eError2; |
| } |
| else |
| { |
| psCtx->ppsAutoHeapArray[uiHeapIndex] = NULL; |
| } |
| } |
| |
| if ((!bDoCheck || (eReturn == PVRSRV_OK)) && psCtx->ppsAutoHeapArray) |
| { |
| OSFreeMem(psCtx->ppsAutoHeapArray); |
| psCtx->ppsAutoHeapArray = NULL; |
| psCtx->uiAutoHeapCount = 0; |
| } |
| |
| return eReturn; |
| } |
| |
| static PVRSRV_ERROR |
| _AllocateMCUFenceAddress(struct _DEVMEM_CONTEXT_ *psCtx) |
| { |
| PVRSRV_ERROR eError; |
| DEVMEM_HEAP *psGeneralHeap; |
| IMG_DEV_VIRTADDR sTempMCUFenceAddr; |
| |
| eError = DevmemFindHeapByName(psCtx, RGX_GENERAL_HEAP_IDENT, &psGeneralHeap); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: General Heap not found (%s)", __func__, GET_ERROR_STRING(eError))); |
| goto e0; |
| } |
| |
| /* MCUFence: Fixed address reserved per Memory Context */ |
| eError = DevmemAllocate(psGeneralHeap, |
| sizeof(IMG_UINT32), |
| RGX_CR_MCU_FENCE_ADDR_ALIGNSIZE, |
| PVRSRV_MEMALLOCFLAG_GPU_READABLE | |
| PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE, |
| "MCUFence", |
| &psCtx->psMCUFenceMemDesc); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate MCU fence word (%s)", __func__, GET_ERROR_STRING(eError))); |
| goto e0; |
| } |
| |
| /* This is the first memory allocation on General Heap so its virtual address |
| * is always equal to heap base address. Storing this address separately is not required. */ |
| eError = DevmemMapToDevice(psCtx->psMCUFenceMemDesc, psGeneralHeap, &sTempMCUFenceAddr); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map MCU fence word (%s)", __func__, GET_ERROR_STRING(eError))); |
| goto e1; |
| } |
| else if (sTempMCUFenceAddr.uiAddr != psGeneralHeap->sBaseAddress.uiAddr) |
| { |
| |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: MCU_FENCE address (%" IMG_UINT64_FMTSPECx ") " |
| "not at the start of General Heap (%" IMG_UINT64_FMTSPECx ")", |
| __func__, sTempMCUFenceAddr.uiAddr, |
| psGeneralHeap->sBaseAddress.uiAddr)); |
| eError = PVRSRV_ERROR_DEVICEMEM_MAP_FAILED; |
| goto e1; |
| } |
| |
| e0: |
| return eError; |
| |
| e1: |
| DevmemFree(psCtx->psMCUFenceMemDesc); |
| psCtx->psMCUFenceMemDesc = NULL; |
| return eError; |
| } |
| |
| /***************************************************************************** |
| * Devmem context functions * |
| *****************************************************************************/ |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemCreateContext(SHARED_DEV_CONNECTION hDevConnection, |
| DEVMEM_HEAPCFGID uiHeapBlueprintID, |
| IMG_BOOL bMCUFenceAllocation, |
| DEVMEM_CONTEXT **ppsCtxPtr) |
| { |
| PVRSRV_ERROR eError; |
| DEVMEM_CONTEXT *psCtx; |
| /* handle to the server-side counterpart of the device memory |
| context (specifically, for handling mapping to device MMU) */ |
| IMG_HANDLE hDevMemServerContext; |
| IMG_HANDLE hPrivData; |
| IMG_BOOL bHeapCfgMetaId = (uiHeapBlueprintID == DEVMEM_HEAPCFG_META); |
| |
| if (ppsCtxPtr == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto e0; |
| } |
| |
| psCtx = OSAllocMem(sizeof *psCtx); |
| if (psCtx == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e0; |
| } |
| |
| psCtx->uiNumHeaps = 0; |
| |
| psCtx->hDevConnection = hDevConnection; |
| |
| /* Create (server-side) Device Memory context */ |
| eError = BridgeDevmemIntCtxCreate(GetBridgeHandle(psCtx->hDevConnection), |
| bHeapCfgMetaId, |
| &hDevMemServerContext, |
| &hPrivData, |
| &psCtx->ui32CPUCacheLineSize); |
| if (eError != PVRSRV_OK) goto e1; |
| |
| psCtx->hDevMemServerContext = hDevMemServerContext; |
| psCtx->hPrivData = hPrivData; |
| |
| /* automagic heap creation */ |
| psCtx->uiAutoHeapCount = 0; |
| |
| eError = _PopulateContextFromBlueprint(psCtx, uiHeapBlueprintID); |
| if (eError != PVRSRV_OK) goto e2; |
| |
| /* Allocate a word at the start of the General heap to be used as MCU_FENCE Address */ |
| if (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORCLIENTS && bMCUFenceAllocation) |
| { |
| eError = _AllocateMCUFenceAddress(psCtx); |
| if (eError != PVRSRV_OK) goto e2; |
| } |
| else |
| { |
| psCtx->psMCUFenceMemDesc = NULL; |
| } |
| |
| *ppsCtxPtr = psCtx; |
| |
| PVR_ASSERT(psCtx->uiNumHeaps == psCtx->uiAutoHeapCount); |
| return PVRSRV_OK; |
| |
| /* error exit paths follow */ |
| |
| e2: |
| PVR_ASSERT(psCtx->uiAutoHeapCount == 0); |
| PVR_ASSERT(psCtx->uiNumHeaps == 0); |
| BridgeDevmemIntCtxDestroy(GetBridgeHandle(psCtx->hDevConnection), hDevMemServerContext); |
| |
| e1: |
| OSFreeMem(psCtx); |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemAcquireDevPrivData(DEVMEM_CONTEXT *psCtx, |
| IMG_HANDLE *hPrivData) |
| { |
| PVRSRV_ERROR eError; |
| |
| if ((psCtx == NULL) || (hPrivData == NULL)) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto e0; |
| } |
| |
| *hPrivData = psCtx->hPrivData; |
| return PVRSRV_OK; |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemReleaseDevPrivData(DEVMEM_CONTEXT *psCtx) |
| { |
| PVRSRV_ERROR eError; |
| |
| if (psCtx == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto e0; |
| } |
| return PVRSRV_OK; |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemFindHeapByName(const struct _DEVMEM_CONTEXT_ *psCtx, |
| const IMG_CHAR *pszHeapName, |
| struct _DEVMEM_HEAP_ **ppsHeapRet) |
| { |
| IMG_UINT32 uiHeapIndex; |
| |
| /* N.B. This func is only useful for finding "automagic" heaps by name */ |
| for (uiHeapIndex = 0; |
| uiHeapIndex < psCtx->uiAutoHeapCount; |
| uiHeapIndex++) |
| { |
| if (!OSStringCompare(psCtx->ppsAutoHeapArray[uiHeapIndex]->pszName, pszHeapName)) |
| { |
| *ppsHeapRet = psCtx->ppsAutoHeapArray[uiHeapIndex]; |
| return PVRSRV_OK; |
| } |
| } |
| |
| return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemDestroyContext(DEVMEM_CONTEXT *psCtx) |
| { |
| PVRSRV_ERROR eError; |
| IMG_BOOL bDoCheck = IMG_TRUE; |
| |
| #if defined(__KERNEL__) |
| PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData(); |
| if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK) |
| { |
| bDoCheck = IMG_FALSE; |
| } |
| #endif |
| |
| if (psCtx == NULL) |
| { |
| return PVRSRV_ERROR_INVALID_PARAMS; |
| } |
| |
| if (psCtx->psMCUFenceMemDesc != NULL) |
| { |
| DevmemReleaseDevVirtAddr(psCtx->psMCUFenceMemDesc); |
| DevmemFree(psCtx->psMCUFenceMemDesc); |
| } |
| |
| eError = _UnpopulateContextFromBlueprint(psCtx); |
| if (bDoCheck && eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: _UnpopulateContextFromBlueprint failed (%d) leaving %d heaps", |
| __func__, eError, psCtx->uiNumHeaps)); |
| goto e1; |
| } |
| |
| eError = BridgeDevmemIntCtxDestroy(GetBridgeHandle(psCtx->hDevConnection), |
| psCtx->hDevMemServerContext); |
| if (bDoCheck && eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: BridgeDevmemIntCtxDestroy failed (%d)", |
| __func__, eError)); |
| goto e1; |
| } |
| |
| /* should be no more heaps left */ |
| if (bDoCheck && psCtx->uiNumHeaps) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Additional heaps remain in DEVMEM_CONTEXT", |
| __func__)); |
| eError = PVRSRV_ERROR_DEVICEMEM_ADDITIONAL_HEAPS_IN_CONTEXT; |
| goto e1; |
| } |
| |
| OSDeviceMemSet(psCtx, 0, sizeof(*psCtx)); |
| OSFreeMem(psCtx); |
| |
| e1: |
| return eError; |
| } |
| |
| /***************************************************************************** |
| * Devmem heap query functions * |
| *****************************************************************************/ |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemHeapConfigCount(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_UINT32 *puiNumHeapConfigsOut) |
| { |
| PVRSRV_ERROR eError; |
| eError = BridgeHeapCfgHeapConfigCount(GetBridgeHandle(hDevConnection), |
| puiNumHeapConfigsOut); |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemHeapCount(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_UINT32 uiHeapConfigIndex, |
| IMG_UINT32 *puiNumHeapsOut) |
| { |
| PVRSRV_ERROR eError; |
| eError = BridgeHeapCfgHeapCount(GetBridgeHandle(hDevConnection), |
| uiHeapConfigIndex, |
| puiNumHeapsOut); |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemHeapConfigName(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_UINT32 uiHeapConfigIndex, |
| IMG_CHAR *pszConfigNameOut, |
| IMG_UINT32 uiConfigNameBufSz) |
| { |
| PVRSRV_ERROR eError; |
| eError = BridgeHeapCfgHeapConfigName(GetBridgeHandle(hDevConnection), |
| uiHeapConfigIndex, |
| uiConfigNameBufSz, |
| pszConfigNameOut); |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemHeapDetails(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_UINT32 uiHeapConfigIndex, |
| IMG_UINT32 uiHeapIndex, |
| IMG_CHAR *pszHeapNameOut, |
| IMG_UINT32 uiHeapNameBufSz, |
| IMG_DEV_VIRTADDR *psDevVAddrBaseOut, |
| IMG_DEVMEM_SIZE_T *puiHeapLengthOut, |
| IMG_UINT32 *puiLog2DataPageSizeOut, |
| IMG_UINT32 *puiLog2ImportAlignmentOut, |
| IMG_UINT32 *puiLog2TilingStrideFactor) |
| { |
| PVRSRV_ERROR eError; |
| |
| eError = BridgeHeapCfgHeapDetails(GetBridgeHandle(hDevConnection), |
| uiHeapConfigIndex, |
| uiHeapIndex, |
| uiHeapNameBufSz, |
| pszHeapNameOut, |
| psDevVAddrBaseOut, |
| puiHeapLengthOut, |
| puiLog2DataPageSizeOut, |
| puiLog2ImportAlignmentOut, |
| puiLog2TilingStrideFactor); |
| |
| VG_MARK_INITIALIZED(pszHeapNameOut,uiHeapNameBufSz); |
| |
| return eError; |
| } |
| |
| /***************************************************************************** |
| * Devmem heap functions * |
| *****************************************************************************/ |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemGetHeapInt(DEVMEM_HEAP *psHeap, |
| IMG_HANDLE *phDevmemHeap) |
| { |
| if (psHeap == NULL) |
| { |
| return PVRSRV_ERROR_INVALID_PARAMS; |
| } |
| *phDevmemHeap = psHeap->hDevMemServerHeap; |
| return PVRSRV_OK; |
| } |
| |
| /* See devicemem.h for important notes regarding the arguments |
| to this function */ |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemCreateHeap(DEVMEM_CONTEXT *psCtx, |
| IMG_DEV_VIRTADDR sBaseAddress, |
| IMG_DEVMEM_SIZE_T uiLength, |
| IMG_UINT32 ui32Log2Quantum, |
| IMG_UINT32 ui32Log2ImportAlignment, |
| IMG_UINT32 ui32Log2TilingStrideFactor, |
| const IMG_CHAR *pszName, |
| DEVMEM_HEAPCFGID uiHeapBlueprintID, |
| DEVMEM_HEAP **ppsHeapPtr) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| PVRSRV_ERROR eError2; |
| DEVMEM_HEAP *psHeap; |
| /* handle to the server-side counterpart of the device memory heap |
| (specifically, for handling mapping to device MMU */ |
| IMG_HANDLE hDevMemServerHeap; |
| IMG_BOOL bRANoSplit = IMG_FALSE; |
| |
| IMG_CHAR aszBuf[100]; |
| IMG_CHAR *pszStr; |
| |
| if (ppsHeapPtr == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto e0; |
| } |
| |
| psHeap = OSAllocMem(sizeof *psHeap); |
| if (psHeap == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e0; |
| } |
| |
| /* Need to keep local copy of heap name, so caller may free theirs */ |
| pszStr = OSAllocMem(OSStringLength(pszName)+1); |
| if (pszStr == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e1; |
| } |
| OSStringCopy(pszStr, pszName); |
| psHeap->pszName = pszStr; |
| |
| psHeap->uiSize = uiLength; |
| psHeap->sBaseAddress = sBaseAddress; |
| OSAtomicWrite(&psHeap->hImportCount,0); |
| |
| OSSNPrintf(aszBuf, sizeof(aszBuf), |
| "NDM heap '%s' (suballocs) ctx:%p", |
| pszName, psCtx); |
| pszStr = OSAllocMem(OSStringLength(aszBuf)+1); |
| if (pszStr == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e2; |
| } |
| OSStringCopy(pszStr, aszBuf); |
| psHeap->pszSubAllocRAName = pszStr; |
| |
| #if defined(PDUMP) |
| /* The META heap is shared globally so a single physical memory import |
| * may be used to satisfy allocations of different processes. |
| * This is problematic when PDumping because the physical memory |
| * import used to satisfy a new allocation may actually have been |
| * imported (and thus the PDump MALLOC generated) before the PDump |
| * client was started, leading to the MALLOC being missing. |
| * |
| * This is solved by disabling splitting of imports for the META physmem |
| * RA, meaning that every firmware allocation gets its own import, thus |
| * ensuring the MALLOC is present for every allocation made within the |
| * pdump capture range |
| */ |
| if (uiHeapBlueprintID == DEVMEM_HEAPCFG_META) |
| { |
| bRANoSplit = IMG_TRUE; |
| } |
| #else |
| PVR_UNREFERENCED_PARAMETER(uiHeapBlueprintID); |
| #endif |
| |
| |
| psHeap->psSubAllocRA = RA_Create(psHeap->pszSubAllocRAName, |
| /* Subsequent imports: */ |
| ui32Log2Quantum, |
| RA_LOCKCLASS_2, |
| _SubAllocImportAlloc, |
| _SubAllocImportFree, |
| (RA_PERARENA_HANDLE) psHeap, |
| bRANoSplit); |
| if (psHeap->psSubAllocRA == NULL) |
| { |
| eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA; |
| goto e3; |
| } |
| |
| psHeap->uiLog2ImportAlignment = ui32Log2ImportAlignment; |
| psHeap->uiLog2TilingStrideFactor = ui32Log2TilingStrideFactor; |
| psHeap->uiLog2Quantum = ui32Log2Quantum; |
| |
| if (!OSStringCompare(pszName, RGX_GENERAL_SVM_HEAP_IDENT)) |
| { |
| /* The SVM heap normally starts out as this type though |
| it may transition to DEVMEM_HEAP_TYPE_USER_MANAGED |
| on platforms with more processor virtual address |
| bits than device virtual address bits */ |
| psHeap->eHeapType = DEVMEM_HEAP_TYPE_KERNEL_MANAGED; |
| } |
| else |
| { |
| psHeap->eHeapType = DEVMEM_HEAP_TYPE_UNKNOWN; |
| } |
| |
| OSSNPrintf(aszBuf, sizeof(aszBuf), |
| "NDM heap '%s' (QVM) ctx:%p", |
| pszName, psCtx); |
| pszStr = OSAllocMem(OSStringLength(aszBuf)+1); |
| if (pszStr == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e4; |
| } |
| OSStringCopy(pszStr, aszBuf); |
| psHeap->pszQuantizedVMRAName = pszStr; |
| |
| psHeap->psQuantizedVMRA = RA_Create(psHeap->pszQuantizedVMRAName, |
| /* Subsequent import: */ |
| 0, RA_LOCKCLASS_1, NULL, NULL, |
| (RA_PERARENA_HANDLE) psHeap, |
| IMG_FALSE); |
| if (psHeap->psQuantizedVMRA == NULL) |
| { |
| eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA; |
| goto e5; |
| } |
| |
| if (!RA_Add(psHeap->psQuantizedVMRA, |
| (RA_BASE_T)sBaseAddress.uiAddr, |
| (RA_LENGTH_T)uiLength, |
| (RA_FLAGS_T)0, /* This RA doesn't use or need flags */ |
| NULL /* per ispan handle */)) |
| { |
| RA_Delete(psHeap->psQuantizedVMRA); |
| eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA; |
| goto e5; |
| } |
| |
| psHeap->psCtx = psCtx; |
| |
| |
| /* Create server-side counterpart of Device Memory heap */ |
| eError = BridgeDevmemIntHeapCreate(GetBridgeHandle(psCtx->hDevConnection), |
| psCtx->hDevMemServerContext, |
| sBaseAddress, |
| uiLength, |
| ui32Log2Quantum, |
| &hDevMemServerHeap); |
| if (eError != PVRSRV_OK) |
| { |
| goto e6; |
| } |
| psHeap->hDevMemServerHeap = hDevMemServerHeap; |
| |
| eError = OSLockCreate(&psHeap->hLock); |
| if (eError != PVRSRV_OK) |
| { |
| goto e7; |
| } |
| |
| psHeap->psCtx->uiNumHeaps ++; |
| *ppsHeapPtr = psHeap; |
| |
| #if defined PVRSRV_NEWDEVMEM_SUPPORT_MEM_TRACKING |
| psHeap->psMemDescList = NULL; |
| #endif /* PVRSRV_NEWDEVMEM_SUPPORT_MEM_TRACKING */ |
| |
| return PVRSRV_OK; |
| |
| /* error exit paths */ |
| e7: |
| eError2 = BridgeDevmemIntHeapDestroy(GetBridgeHandle(psCtx->hDevConnection), |
| psHeap->hDevMemServerHeap); |
| PVR_ASSERT (eError2 == PVRSRV_OK); |
| e6: |
| if (psHeap->psQuantizedVMRA) |
| RA_Delete(psHeap->psQuantizedVMRA); |
| e5: |
| if (psHeap->pszQuantizedVMRAName) |
| OSFreeMem(psHeap->pszQuantizedVMRAName); |
| e4: |
| RA_Delete(psHeap->psSubAllocRA); |
| e3: |
| OSFreeMem(psHeap->pszSubAllocRAName); |
| e2: |
| OSFreeMem(psHeap->pszName); |
| e1: |
| OSFreeMem(psHeap); |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemGetHeapBaseDevVAddr(struct _DEVMEM_HEAP_ *psHeap, |
| IMG_DEV_VIRTADDR *pDevVAddr) |
| { |
| if (psHeap == NULL) |
| { |
| return PVRSRV_ERROR_INVALID_PARAMS; |
| } |
| |
| *pDevVAddr = psHeap->sBaseAddress; |
| |
| return PVRSRV_OK; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemExportalignAdjustSizeAndAlign(IMG_UINT32 uiLog2Quantum, |
| IMG_DEVMEM_SIZE_T *puiSize, |
| IMG_DEVMEM_ALIGN_T *puiAlign) |
| { |
| IMG_DEVMEM_SIZE_T uiSize = *puiSize; |
| IMG_DEVMEM_ALIGN_T uiAlign = *puiAlign; |
| |
| /* Just in case someone changes definition of IMG_DEVMEM_ALIGN_T. */ |
| static_assert(sizeof(unsigned long long) == sizeof(uiAlign), |
| "invalid uiAlign size"); |
| /* This value is used for shifting so it cannot be greater than number |
| * of bits in unsigned long long (sizeof(1ULL)). Using greater value is |
| * undefined behaviour. */ |
| if (uiLog2Quantum >= sizeof(unsigned long long) * 8) |
| { |
| return PVRSRV_ERROR_INVALID_PARAMS; |
| } |
| |
| if ((1ULL << uiLog2Quantum) > uiAlign) |
| { |
| uiAlign = 1ULL << uiLog2Quantum; |
| } |
| uiSize = (uiSize + uiAlign - 1) & ~(uiAlign - 1); |
| |
| *puiSize = uiSize; |
| *puiAlign = uiAlign; |
| |
| return PVRSRV_OK; |
| } |
| |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemDestroyHeap(DEVMEM_HEAP *psHeap) |
| { |
| PVRSRV_ERROR eError; |
| IMG_INT uiImportCount; |
| #if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE) |
| IMG_BOOL bDoCheck = IMG_TRUE; |
| #if defined(__KERNEL__) |
| if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK) |
| { |
| bDoCheck = IMG_FALSE; |
| } |
| #endif |
| #endif |
| |
| if (psHeap == NULL) |
| { |
| return PVRSRV_ERROR_INVALID_PARAMS; |
| } |
| |
| uiImportCount = OSAtomicRead(&psHeap->hImportCount); |
| if (uiImportCount > 0) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%d(%s) leaks remain", uiImportCount, psHeap->pszName)); |
| #if defined(__KERNEL__) |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| PVR_DPF((PVR_DBG_ERROR, "Details of remaining allocated device memory (for all processes):")); |
| RIDumpAllKM(); |
| #else |
| PVR_DPF((PVR_DBG_ERROR, "Compile with PVRSRV_ENABLE_GPU_MEMORY_INFO=1 to get a full " |
| "list of all driver allocations.")); |
| #endif |
| #endif |
| #if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE) |
| if (bDoCheck) |
| #endif |
| { |
| return PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP; |
| } |
| } |
| |
| eError = BridgeDevmemIntHeapDestroy(GetBridgeHandle(psHeap->psCtx->hDevConnection), |
| psHeap->hDevMemServerHeap); |
| #if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE) |
| if (bDoCheck) |
| #endif |
| { |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: BridgeDevmemIntHeapDestroy failed (%d)", |
| __func__, eError)); |
| return eError; |
| } |
| } |
| |
| PVR_ASSERT(psHeap->psCtx->uiNumHeaps > 0); |
| psHeap->psCtx->uiNumHeaps--; |
| |
| OSLockDestroy(psHeap->hLock); |
| |
| if (psHeap->psQuantizedVMRA) |
| { |
| RA_Delete(psHeap->psQuantizedVMRA); |
| } |
| if (psHeap->pszQuantizedVMRAName) |
| { |
| OSFreeMem(psHeap->pszQuantizedVMRAName); |
| } |
| |
| RA_Delete(psHeap->psSubAllocRA); |
| OSFreeMem(psHeap->pszSubAllocRAName); |
| |
| OSFreeMem(psHeap->pszName); |
| |
| OSDeviceMemSet(psHeap, 0, sizeof(*psHeap)); |
| OSFreeMem(psHeap); |
| |
| return PVRSRV_OK; |
| } |
| |
| /***************************************************************************** |
| * Devmem allocation/free functions * |
| *****************************************************************************/ |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemSubAllocate(IMG_UINT8 uiPreAllocMultiplier, |
| DEVMEM_HEAP *psHeap, |
| IMG_DEVMEM_SIZE_T uiSize, |
| IMG_DEVMEM_ALIGN_T uiAlign, |
| DEVMEM_FLAGS_T uiFlags, |
| const IMG_CHAR *pszText, |
| DEVMEM_MEMDESC **ppsMemDescPtr) |
| { |
| RA_BASE_T uiAllocatedAddr = 0; |
| RA_LENGTH_T uiAllocatedSize; |
| RA_PERISPAN_HANDLE hImport; /* the "import" from which this sub-allocation came */ |
| PVRSRV_ERROR eError; |
| DEVMEM_MEMDESC *psMemDesc = NULL; |
| IMG_DEVMEM_OFFSET_T uiOffset = 0; |
| DEVMEM_IMPORT *psImport; |
| IMG_UINT32 ui32CPUCacheLineSize; |
| void *pvAddr = NULL; |
| |
| IMG_BOOL bImportClean; |
| IMG_BOOL bCPUCleanFlag = PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags); |
| IMG_BOOL bZero = PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags); |
| IMG_BOOL bPoisonOnAlloc = PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags); |
| IMG_BOOL bCPUCached = (PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) || |
| PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags)); |
| IMG_BOOL bGPUCached = (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) || |
| PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags)); |
| PVRSRV_CACHE_OP eOp = PVRSRV_CACHE_OP_INVALIDATE; |
| IMG_UINT32 ui32CacheLineSize = 0; |
| |
| if (uiFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC) |
| { |
| /* Deferred Allocation not supported on SubAllocs*/ |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto failParams; |
| } |
| |
| if (psHeap == NULL || psHeap->psCtx == NULL ||ppsMemDescPtr == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto failParams; |
| } |
| |
| #if defined(__KERNEL__) |
| { |
| /* The hDevConnection holds two different types of pointers depending on the |
| * address space in which it is used. |
| * In this instance the variable points to the device node in server */ |
| PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection; |
| ui32CacheLineSize = GET_ROGUE_CACHE_LINE_SIZE(PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, SLC_CACHE_LINE_SIZE_BITS)); |
| } |
| #else |
| ui32CacheLineSize = ROGUE_CACHE_LINE_SIZE; |
| #endif |
| |
| /* The following logic makes sure that any cached memory is aligned to both the CPU and GPU. |
| * To be aligned on both you have to take the Lowest Common Multiple (LCM) of the cache line sizes of each. |
| * As the possibilities are all powers of 2 then simply the largest number can be picked as the LCM. |
| * Therefore this algorithm just picks the highest from the CPU, GPU and given alignments. |
| */ |
| ui32CPUCacheLineSize = psHeap->psCtx->ui32CPUCacheLineSize; |
| /* If the CPU cache line size is larger than the alignment given then it is the lowest common multiple |
| * Also checking if the allocation is going to be cached on the CPU |
| * Currently there is no check for the validity of the cache coherent option. |
| * In this case, the alignment could be applied but the mode could still fall back to uncached. |
| */ |
| if (ui32CPUCacheLineSize > uiAlign && bCPUCached) |
| { |
| uiAlign = ui32CPUCacheLineSize; |
| } |
| |
| /* If the GPU cache line size is larger than the alignment given then it is the lowest common multiple |
| * Also checking if the allocation is going to be cached on the GPU via checking for any of the cached options. |
| * Currently there is no check for the validity of the cache coherent option. |
| * In this case, the alignment could be applied but the mode could still fall back to uncached. |
| */ |
| if (ui32CacheLineSize > uiAlign && bGPUCached) |
| { |
| uiAlign = ui32CacheLineSize; |
| } |
| |
| eError = _DevmemValidateParams(uiSize, |
| uiAlign, |
| &uiFlags); |
| if (eError != PVRSRV_OK) |
| { |
| goto failParams; |
| } |
| |
| eError =_DevmemMemDescAlloc(&psMemDesc); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMemDescAlloc; |
| } |
| |
| /* No request for exportable memory so use the RA */ |
| eError = RA_Alloc(psHeap->psSubAllocRA, |
| uiSize, |
| uiPreAllocMultiplier, |
| uiFlags, |
| uiAlign, |
| pszText, |
| &uiAllocatedAddr, |
| &uiAllocatedSize, |
| &hImport); |
| if (PVRSRV_OK != eError) |
| { |
| goto failDeviceMemAlloc; |
| } |
| |
| psImport = hImport; |
| |
| /* This assignment is assuming the RA returns an hImport where suballocations |
| * can be made from if uiSize is NOT a page multiple of the passed heap. |
| * |
| * So we check if uiSize is a page multiple and mark it as exportable |
| * if it is not. |
| * */ |
| if (!(uiSize & ((1ULL << psHeap->uiLog2Quantum) - 1)) && |
| (uiPreAllocMultiplier == RA_NO_IMPORT_MULTIPLIER)) |
| { |
| psImport->uiProperties |= DEVMEM_PROPERTIES_EXPORTABLE; |
| } |
| psImport->uiProperties |= DEVMEM_PROPERTIES_SUBALLOCATABLE; |
| |
| uiOffset = uiAllocatedAddr - psImport->sDeviceImport.sDevVAddr.uiAddr; |
| |
| #if defined(PDUMP) && defined(DEBUG) |
| #if defined(__KERNEL__) |
| PDUMPCOMMENTWITHFLAGS(PDUMP_CONT, |
| "Suballocated %u Byte for \"%s\" from PMR with handle ID: 0x%p (PID %u)", |
| (IMG_UINT32) uiSize, pszText, psImport->hPMR, OSGetCurrentProcessID()); |
| #else |
| PDUMPCOMMENTF(psHeap->psCtx->hDevConnection, PDUMP_FLAGS_CONTINUOUS, |
| "Suballocated %u Byte for \"%s\" from PMR with handle ID: %p (PID %u)", |
| (IMG_UINT32) uiSize, |
| pszText, |
| psImport->hPMR, |
| OSGetCurrentProcessID()); |
| #endif |
| #endif |
| |
| _DevmemMemDescInit(psMemDesc, |
| uiOffset, |
| psImport, |
| uiSize); |
| |
| bImportClean = ((psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_CLEAN) != 0); |
| |
| /* Zero the memory */ |
| if (bZero) |
| { |
| /* Has the import been zeroed on allocation and were no suballocations returned to it so far? */ |
| bImportClean = bImportClean && ((psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_ZEROED) != 0); |
| |
| if (!bImportClean) |
| { |
| eOp = PVRSRV_CACHE_OP_FLUSH; |
| |
| eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMaintenance; |
| } |
| |
| /* uiSize is a 64-bit quantity whereas the 3rd argument |
| * to OSDeviceMemSet is a 32-bit quantity on 32-bit systems |
| * hence a compiler warning of implicit cast and loss of data. |
| * Added explicit cast and assert to remove warning. |
| */ |
| PVR_ASSERT(uiSize < IMG_UINT32_MAX); |
| |
| OSDeviceMemSet(pvAddr, 0x0, (size_t) uiSize); |
| #if defined(PDUMP) |
| DevmemPDumpLoadZeroMem(psMemDesc, 0, uiSize, PDUMP_FLAGS_CONTINUOUS); |
| #endif |
| } |
| } |
| else if (bPoisonOnAlloc) |
| { |
| /* Has the import been poisoned on allocation and were no suballocations returned to it so far? */ |
| bPoisonOnAlloc = (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_POISONED) != 0; |
| |
| if (!bPoisonOnAlloc) |
| { |
| eOp = PVRSRV_CACHE_OP_FLUSH; |
| |
| eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMaintenance; |
| } |
| |
| if (PVRSRV_CHECK_CPU_UNCACHED(uiFlags) || |
| PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags)) |
| { |
| OSDeviceMemSet(pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE, |
| uiSize); |
| } |
| else |
| { |
| OSCachedMemSet(pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE, |
| uiSize); |
| } |
| |
| bPoisonOnAlloc = IMG_TRUE; |
| } |
| } |
| |
| /* Flush or invalidate */ |
| if (bCPUCached && !bImportClean && (bZero || bCPUCleanFlag || bPoisonOnAlloc)) |
| { |
| /* BridgeCacheOpQueue _may_ be deferred so use BridgeCacheOpExec |
| to ensure this cache maintenance is actioned immediately */ |
| eError = BridgeCacheOpExec (GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->psImport->hPMR, |
| (IMG_UINT64)(uintptr_t) |
| pvAddr - psMemDesc->uiOffset, |
| psMemDesc->uiOffset, |
| psMemDesc->uiAllocSize, |
| eOp); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMaintenance; |
| } |
| } |
| |
| if (pvAddr) |
| { |
| DevmemReleaseCpuVirtAddr(psMemDesc); |
| pvAddr = NULL; |
| } |
| |
| /* copy the allocation descriptive name and size so it can be passed to DevicememHistory when |
| * the allocation gets mapped/unmapped |
| */ |
| _CheckAnnotationLength(pszText); |
| #if defined(__KERNEL__) |
| OSStringLCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN); |
| #else |
| OSStringNCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN); |
| psMemDesc->szText[DEVMEM_ANNOTATION_MAX_LEN - 1] = '\0'; |
| #endif /* if defined(__KERNEL__) */ |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI)) |
| { |
| /* Attach RI information */ |
| eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->psImport->hPMR, |
| OSStringNLength(psMemDesc->szText, DEVMEM_ANNOTATION_MAX_LEN), |
| psMemDesc->szText, |
| psMemDesc->uiOffset, |
| uiAllocatedSize, |
| IMG_FALSE, |
| IMG_TRUE, |
| &(psMemDesc->hRIHandle)); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (Error=%d)", __func__, eError)); |
| } |
| } |
| #else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| PVR_UNREFERENCED_PARAMETER (pszText); |
| #endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| |
| *ppsMemDescPtr = psMemDesc; |
| |
| return PVRSRV_OK; |
| |
| /* error exit paths follow */ |
| |
| failMaintenance: |
| if (pvAddr) |
| { |
| DevmemReleaseCpuVirtAddr(psMemDesc); |
| pvAddr = NULL; |
| } |
| _DevmemMemDescRelease(psMemDesc); |
| psMemDesc = NULL; /* Make sure we don't do a discard after the release */ |
| failDeviceMemAlloc: |
| if (psMemDesc) |
| { |
| _DevmemMemDescDiscard(psMemDesc); |
| } |
| failMemDescAlloc: |
| failParams: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC, |
| __func__, |
| PVRSRVGETERRORSTRING(eError), |
| uiSize)); |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemAllocateExportable(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_DEVMEM_SIZE_T uiSize, |
| IMG_DEVMEM_ALIGN_T uiAlign, |
| IMG_UINT32 uiLog2HeapPageSize, |
| DEVMEM_FLAGS_T uiFlags, |
| const IMG_CHAR *pszText, |
| DEVMEM_MEMDESC **ppsMemDescPtr) |
| { |
| PVRSRV_ERROR eError; |
| DEVMEM_MEMDESC *psMemDesc = NULL; |
| DEVMEM_IMPORT *psImport; |
| IMG_UINT32 ui32MappingTable = 0; |
| |
| eError = DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize, |
| &uiSize, |
| &uiAlign); |
| if (eError != PVRSRV_OK) |
| { |
| goto failParams; |
| } |
| |
| eError = _DevmemValidateParams(uiSize, |
| uiAlign, |
| &uiFlags); |
| if (eError != PVRSRV_OK) |
| { |
| goto failParams; |
| } |
| |
| eError =_DevmemMemDescAlloc(&psMemDesc); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMemDescAlloc; |
| } |
| |
| eError = _AllocateDeviceMemory(hDevConnection, |
| uiLog2HeapPageSize, |
| uiSize, |
| uiSize, |
| 1, |
| 1, |
| &ui32MappingTable, |
| uiAlign, |
| uiFlags, |
| IMG_TRUE, |
| pszText, |
| &psImport); |
| if (eError != PVRSRV_OK) |
| { |
| goto failDeviceMemAlloc; |
| } |
| |
| _DevmemMemDescInit(psMemDesc, |
| 0, |
| psImport, |
| uiSize); |
| |
| *ppsMemDescPtr = psMemDesc; |
| |
| /* copy the allocation descriptive name and size so it can be passed to DevicememHistory when |
| * the allocation gets mapped/unmapped |
| */ |
| _CheckAnnotationLength(pszText); |
| #if defined(__KERNEL__) |
| OSStringLCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN); |
| #else |
| OSStringNCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN); |
| psMemDesc->szText[DEVMEM_ANNOTATION_MAX_LEN - 1] = '\0'; |
| #endif /* if defined(__KERNEL__) */ |
| |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI)) |
| { |
| eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection), |
| psImport->hPMR); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntry failed (Error=%d)", __func__, eError)); |
| } |
| |
| /* Attach RI information */ |
| eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psImport->hDevConnection), |
| psImport->hPMR, |
| sizeof("^"), |
| "^", |
| psMemDesc->uiOffset, |
| uiSize, |
| IMG_FALSE, |
| IMG_FALSE, |
| &psMemDesc->hRIHandle); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (Error=%d)", __func__, eError)); |
| } |
| } |
| #else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| PVR_UNREFERENCED_PARAMETER (pszText); |
| #endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| |
| return PVRSRV_OK; |
| |
| /* error exit paths follow */ |
| |
| failDeviceMemAlloc: |
| _DevmemMemDescDiscard(psMemDesc); |
| |
| failMemDescAlloc: |
| failParams: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC, |
| __func__, |
| PVRSRVGETERRORSTRING(eError), |
| uiSize)); |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemAllocateSparse(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_DEVMEM_SIZE_T uiSize, |
| IMG_DEVMEM_SIZE_T uiChunkSize, |
| IMG_UINT32 ui32NumPhysChunks, |
| IMG_UINT32 ui32NumVirtChunks, |
| IMG_UINT32 *pui32MappingTable, |
| IMG_DEVMEM_ALIGN_T uiAlign, |
| IMG_UINT32 uiLog2HeapPageSize, |
| DEVMEM_FLAGS_T uiFlags, |
| const IMG_CHAR *pszText, |
| DEVMEM_MEMDESC **ppsMemDescPtr) |
| { |
| PVRSRV_ERROR eError; |
| DEVMEM_MEMDESC *psMemDesc = NULL; |
| DEVMEM_IMPORT *psImport; |
| |
| eError = DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize, |
| &uiSize, |
| &uiAlign); |
| if (eError != PVRSRV_OK) |
| { |
| goto failParams; |
| } |
| |
| eError = _DevmemValidateParams(uiSize, |
| uiAlign, |
| &uiFlags); |
| if (eError != PVRSRV_OK) |
| { |
| goto failParams; |
| } |
| |
| eError =_DevmemMemDescAlloc(&psMemDesc); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMemDescAlloc; |
| } |
| |
| eError = _AllocateDeviceMemory(hDevConnection, |
| uiLog2HeapPageSize, |
| uiSize, |
| uiChunkSize, |
| ui32NumPhysChunks, |
| ui32NumVirtChunks, |
| pui32MappingTable, |
| uiAlign, |
| uiFlags, |
| IMG_TRUE, |
| pszText, |
| &psImport); |
| if (eError != PVRSRV_OK) |
| { |
| goto failDeviceMemAlloc; |
| } |
| |
| _DevmemMemDescInit(psMemDesc, |
| 0, |
| psImport, |
| uiSize); |
| |
| /* copy the allocation descriptive name and size so it can be passed to DevicememHistory when |
| * the allocation gets mapped/unmapped |
| */ |
| _CheckAnnotationLength(pszText); |
| #if defined(__KERNEL__) |
| OSStringLCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN); |
| #else |
| OSStringNCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN); |
| psMemDesc->szText[DEVMEM_ANNOTATION_MAX_LEN - 1] = '\0'; |
| #endif /* if defined(__KERNEL__) */ |
| |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI)) |
| { |
| eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection), |
| psImport->hPMR); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntry failed (Error=%d)", __func__, eError)); |
| } |
| |
| /* Attach RI information */ |
| eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->psImport->hPMR, |
| sizeof("^"), |
| "^", |
| psMemDesc->uiOffset, |
| uiSize, |
| IMG_FALSE, |
| IMG_FALSE, |
| &psMemDesc->hRIHandle); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (Error=%d)", __func__, eError)); |
| } |
| } |
| #else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| PVR_UNREFERENCED_PARAMETER (pszText); |
| #endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| |
| *ppsMemDescPtr = psMemDesc; |
| |
| return PVRSRV_OK; |
| |
| /* error exit paths follow */ |
| |
| failDeviceMemAlloc: |
| _DevmemMemDescDiscard(psMemDesc); |
| |
| failMemDescAlloc: |
| failParams: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC, |
| __func__, |
| PVRSRVGETERRORSTRING(eError), |
| uiSize)); |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemMakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_HANDLE hServerHandle, |
| IMG_HANDLE *hLocalImportHandle) |
| { |
| return BridgePMRMakeLocalImportHandle(GetBridgeHandle(hDevConnection), |
| hServerHandle, |
| hLocalImportHandle); |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemUnmakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_HANDLE hLocalImportHandle) |
| { |
| return BridgePMRUnmakeLocalImportHandle(GetBridgeHandle(hDevConnection), hLocalImportHandle); |
| } |
| |
| /***************************************************************************** |
| * Devmem unsecure export functions * |
| *****************************************************************************/ |
| |
| #if defined(SUPPORT_INSECURE_EXPORT) |
| |
| static PVRSRV_ERROR |
| _Mapping_Export(DEVMEM_IMPORT *psImport, |
| DEVMEM_EXPORTHANDLE *phPMRExportHandlePtr, |
| DEVMEM_EXPORTKEY *puiExportKeyPtr, |
| DEVMEM_SIZE_T *puiSize, |
| DEVMEM_LOG2ALIGN_T *puiLog2Contig) |
| { |
| /* Gets an export handle and key for the PMR used for this mapping */ |
| /* Can only be done if there are no suballocations for this mapping */ |
| |
| PVRSRV_ERROR eError; |
| DEVMEM_EXPORTHANDLE hPMRExportHandle; |
| DEVMEM_EXPORTKEY uiExportKey; |
| IMG_DEVMEM_SIZE_T uiSize; |
| IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig; |
| |
| if (psImport == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto failParams; |
| } |
| |
| if ((psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE) == 0) |
| { |
| eError = PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION; |
| goto failParams; |
| } |
| |
| eError = BridgePMRExportPMR(GetBridgeHandle(psImport->hDevConnection), |
| psImport->hPMR, |
| &hPMRExportHandle, |
| &uiSize, |
| &uiLog2Contig, |
| &uiExportKey); |
| if (eError != PVRSRV_OK) |
| { |
| goto failExport; |
| } |
| |
| PVR_ASSERT(uiSize == psImport->uiSize); |
| |
| *phPMRExportHandlePtr = hPMRExportHandle; |
| *puiExportKeyPtr = uiExportKey; |
| *puiSize = uiSize; |
| *puiLog2Contig = uiLog2Contig; |
| |
| return PVRSRV_OK; |
| |
| /* error exit paths follow */ |
| |
| failExport: |
| failParams: |
| |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| |
| } |
| |
| static void |
| _Mapping_Unexport(DEVMEM_IMPORT *psImport, |
| DEVMEM_EXPORTHANDLE hPMRExportHandle) |
| { |
| PVRSRV_ERROR eError; |
| |
| PVR_ASSERT (psImport != NULL); |
| |
| eError = BridgePMRUnexportPMR(GetBridgeHandle(psImport->hDevConnection), |
| hPMRExportHandle); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemExport(DEVMEM_MEMDESC *psMemDesc, |
| DEVMEM_EXPORTCOOKIE *psExportCookie) |
| { |
| /* Caller to provide storage for export cookie struct */ |
| PVRSRV_ERROR eError; |
| IMG_HANDLE hPMRExportHandle = 0; |
| IMG_UINT64 uiPMRExportPassword = 0; |
| IMG_DEVMEM_SIZE_T uiSize = 0; |
| IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig = 0; |
| |
| if (psMemDesc == NULL || psExportCookie == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto e0; |
| } |
| |
| if (DEVMEM_PROPERTIES_EXPORTABLE != |
| (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: This Memory (0x%p) cannot be exported!...", |
| __func__, psMemDesc)); |
| eError = PVRSRV_ERROR_INVALID_REQUEST; |
| goto e0; |
| } |
| |
| eError = _Mapping_Export(psMemDesc->psImport, |
| &hPMRExportHandle, |
| &uiPMRExportPassword, |
| &uiSize, |
| &uiLog2Contig); |
| if (eError != PVRSRV_OK) |
| { |
| psExportCookie->uiSize = 0; |
| goto e0; |
| } |
| |
| psExportCookie->hPMRExportHandle = hPMRExportHandle; |
| psExportCookie->uiPMRExportPassword = uiPMRExportPassword; |
| psExportCookie->uiSize = uiSize; |
| psExportCookie->uiLog2ContiguityGuarantee = uiLog2Contig; |
| |
| return PVRSRV_OK; |
| |
| /* error exit paths follow */ |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| IMG_INTERNAL void |
| DevmemUnexport(DEVMEM_MEMDESC *psMemDesc, |
| DEVMEM_EXPORTCOOKIE *psExportCookie) |
| { |
| _Mapping_Unexport(psMemDesc->psImport, |
| psExportCookie->hPMRExportHandle); |
| |
| psExportCookie->uiSize = 0; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemImport(SHARED_DEV_CONNECTION hDevConnection, |
| DEVMEM_EXPORTCOOKIE *psCookie, |
| DEVMEM_FLAGS_T uiFlags, |
| DEVMEM_MEMDESC **ppsMemDescPtr) |
| { |
| DEVMEM_MEMDESC *psMemDesc = NULL; |
| DEVMEM_IMPORT *psImport; |
| IMG_HANDLE hPMR; |
| PVRSRV_ERROR eError; |
| |
| if (ppsMemDescPtr == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto failParams; |
| } |
| |
| eError =_DevmemMemDescAlloc(&psMemDesc); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMemDescAlloc; |
| } |
| |
| eError = _DevmemImportStructAlloc(hDevConnection, |
| &psImport); |
| if (eError != PVRSRV_OK) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto failImportAlloc; |
| } |
| |
| /* Get a handle to the PMR (inc refcount) */ |
| eError = BridgePMRImportPMR(GetBridgeHandle(hDevConnection), |
| psCookie->hPMRExportHandle, |
| psCookie->uiPMRExportPassword, |
| psCookie->uiSize, /* not trusted - just for sanity checks */ |
| psCookie->uiLog2ContiguityGuarantee, /* not trusted - just for sanity checks */ |
| &hPMR); |
| if (eError != PVRSRV_OK) |
| { |
| goto failImport; |
| } |
| |
| _DevmemImportStructInit(psImport, |
| psCookie->uiSize, |
| 1ULL << psCookie->uiLog2ContiguityGuarantee, |
| uiFlags, |
| hPMR, |
| DEVMEM_PROPERTIES_IMPORTED | |
| DEVMEM_PROPERTIES_EXPORTABLE); |
| |
| _DevmemMemDescInit(psMemDesc, |
| 0, |
| psImport, |
| psImport->uiSize); |
| |
| *ppsMemDescPtr = psMemDesc; |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI)) |
| { |
| /* Attach RI information */ |
| eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->psImport->hPMR, |
| sizeof("^"), |
| "^", |
| psMemDesc->uiOffset, |
| psMemDesc->psImport->uiSize, |
| IMG_TRUE, |
| IMG_TRUE, |
| &psMemDesc->hRIHandle); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (Error=%d)", __func__, eError)); |
| } |
| } |
| #endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| |
| return PVRSRV_OK; |
| |
| /* error exit paths follow */ |
| |
| failImport: |
| _DevmemImportDiscard(psImport); |
| failImportAlloc: |
| _DevmemMemDescDiscard(psMemDesc); |
| failMemDescAlloc: |
| failParams: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| |
| return eError; |
| } |
| |
| #endif /* SUPPORT_INSECURE_EXPORT */ |
| |
| /***************************************************************************** |
| * Common MemDesc functions * |
| *****************************************************************************/ |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemUnpin(DEVMEM_MEMDESC *psMemDesc) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| DEVMEM_IMPORT *psImport = psMemDesc->psImport; |
| |
| if (psImport->uiProperties & DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE) |
| { |
| eError = PVRSRV_ERROR_INVALID_REQUEST; |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: The passed allocation is not valid to unpin", |
| __func__)); |
| |
| goto e_exit; |
| } |
| |
| /* Stop if the allocation might have suballocations. */ |
| if (!(psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE)) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: The passed allocation is not valid to unpin because " |
| "there might be suballocations on it. Make sure you allocate a page multiple " |
| "of the heap when using PVRSRVAllocDeviceMem()", |
| __func__)); |
| |
| goto e_exit; |
| } |
| |
| /* Stop if the Import is still mapped to CPU */ |
| if (psImport->sCPUImport.ui32RefCount) |
| { |
| eError = PVRSRV_ERROR_STILL_MAPPED; |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: There are still %u references on the CPU mapping. " |
| "Please remove all CPU mappings before unpinning.", |
| __func__, |
| psImport->sCPUImport.ui32RefCount)); |
| |
| goto e_exit; |
| } |
| |
| /* Only unpin if it is not already unpinned |
| * Return PVRSRV_OK */ |
| if (psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED) |
| { |
| goto e_exit; |
| } |
| |
| /* Unpin it and invalidate mapping */ |
| if (psImport->sDeviceImport.bMapped) |
| { |
| eError = BridgeDevmemIntUnpinInvalidate(GetBridgeHandle(psImport->hDevConnection), |
| psImport->sDeviceImport.hMapping, |
| psImport->hPMR); |
| } |
| else |
| { |
| /* Or just unpin it */ |
| eError = BridgeDevmemIntUnpin(GetBridgeHandle(psImport->hDevConnection), |
| psImport->hPMR); |
| } |
| |
| /* Update flags and RI when call was successful */ |
| if (eError == PVRSRV_OK) |
| { |
| psImport->uiProperties |= DEVMEM_PROPERTIES_UNPINNED; |
| } |
| else |
| { |
| /* Or just show what went wrong */ |
| PVR_DPF((PVR_DBG_ERROR, "%s: Unpin aborted because of error %d", |
| __func__, |
| eError)); |
| } |
| |
| e_exit: |
| return eError; |
| } |
| |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemPin(DEVMEM_MEMDESC *psMemDesc) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| DEVMEM_IMPORT *psImport = psMemDesc->psImport; |
| |
| if (psImport->uiProperties & DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE) |
| { |
| eError = PVRSRV_ERROR_INVALID_REQUEST; |
| goto e_exit; |
| } |
| |
| /* Only pin if it is unpinned */ |
| if ((psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED) == 0) |
| { |
| goto e_exit; |
| } |
| |
| /* Pin it and make mapping valid */ |
| if (psImport->sDeviceImport.bMapped) |
| { |
| eError = BridgeDevmemIntPinValidate(GetBridgeHandle(psImport->hDevConnection), |
| psImport->sDeviceImport.hMapping, |
| psImport->hPMR); |
| } |
| else |
| { |
| /* Or just pin it */ |
| eError = BridgeDevmemIntPin(GetBridgeHandle(psImport->hDevConnection), |
| psImport->hPMR); |
| } |
| |
| if ((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_PMR_NEW_MEMORY)) |
| { |
| psImport->uiProperties &= ~DEVMEM_PROPERTIES_UNPINNED; |
| } |
| else |
| { |
| /* Or just show what went wrong */ |
| PVR_DPF((PVR_DBG_ERROR, "%s: Pin aborted because of error %d", |
| __func__, |
| eError)); |
| } |
| |
| e_exit: |
| return eError; |
| } |
| |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemGetSize(DEVMEM_MEMDESC *psMemDesc, IMG_DEVMEM_SIZE_T* puiSize) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| |
| *puiSize = psMemDesc->uiAllocSize; |
| |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemGetAnnotation(DEVMEM_MEMDESC *psMemDesc, IMG_CHAR **pszAnnotation) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| |
| *pszAnnotation = psMemDesc->szText; |
| |
| return eError; |
| } |
| |
| /* |
| This function is called for freeing any class of memory |
| */ |
| IMG_INTERNAL IMG_BOOL |
| DevmemFree(DEVMEM_MEMDESC *psMemDesc) |
| { |
| if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_SECURE) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Please use methods dedicated to secure buffers.", |
| __func__)); |
| return IMG_FALSE; |
| } |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI)) |
| { |
| if (psMemDesc->hRIHandle) |
| { |
| PVRSRV_ERROR eError; |
| |
| eError = BridgeRIDeleteMEMDESCEntry(GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->hRIHandle); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIDeleteMEMDESCEntry failed (Error=%d)", __func__, eError)); |
| } |
| } |
| } |
| #endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| |
| return _DevmemMemDescRelease(psMemDesc); |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemMapToDevice(DEVMEM_MEMDESC *psMemDesc, |
| DEVMEM_HEAP *psHeap, |
| IMG_DEV_VIRTADDR *psDevVirtAddr) |
| { |
| DEVMEM_IMPORT *psImport; |
| IMG_DEV_VIRTADDR sDevVAddr; |
| PVRSRV_ERROR eError; |
| IMG_BOOL bMap = IMG_TRUE; |
| IMG_BOOL bDestroyed = IMG_FALSE; |
| IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS; |
| |
| /* Do not try to map unpinned memory */ |
| if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED) |
| { |
| eError = PVRSRV_ERROR_INVALID_MAP_REQUEST; |
| goto failFlags; |
| } |
| |
| OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock); |
| if (psHeap == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto failParams; |
| } |
| |
| if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0) |
| { |
| eError = PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED; |
| goto failCheck; |
| } |
| |
| /* Don't map memory for deferred allocations */ |
| if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC) |
| { |
| PVR_ASSERT(psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE); |
| bMap = IMG_FALSE; |
| } |
| |
| DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d", |
| __func__, |
| psMemDesc, |
| psMemDesc->sDeviceMemDesc.ui32RefCount, |
| psMemDesc->sDeviceMemDesc.ui32RefCount+1); |
| |
| psImport = psMemDesc->psImport; |
| _DevmemMemDescAcquire(psMemDesc); |
| |
| #if defined(__KERNEL__) |
| { |
| PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection; |
| PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDevNode->pvDevice; |
| |
| if (((psHeap == psDevInfo->psFirmwareMainHeap) || |
| (psHeap == psDevInfo->psFirmwareConfigHeap)) && |
| PVRSRV_VZ_MODE_IS(DRIVER_MODE_GUEST)) |
| { |
| ui64OptionalMapAddress = _GuestFWHeapVA(psImport->hPMR, psDevNode); |
| } |
| } |
| #endif |
| |
| eError = _DevmemImportStructDevMap(psHeap, |
| bMap, |
| psImport, |
| ui64OptionalMapAddress); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMap; |
| } |
| |
| sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr; |
| sDevVAddr.uiAddr += psMemDesc->uiOffset; |
| psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr; |
| psMemDesc->sDeviceMemDesc.ui32RefCount++; |
| |
| *psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr; |
| |
| OSLockRelease(psMemDesc->sDeviceMemDesc.hLock); |
| |
| if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) |
| { |
| BridgeDevicememHistoryMap(GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->psImport->hPMR, |
| psMemDesc->uiOffset, |
| psMemDesc->sDeviceMemDesc.sDevVAddr, |
| psMemDesc->uiAllocSize, |
| psMemDesc->szText, |
| DevmemGetHeapLog2PageSize(psHeap), |
| psMemDesc->ui32AllocationIndex, |
| &psMemDesc->ui32AllocationIndex); |
| } |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI)) |
| { |
| if (psMemDesc->hRIHandle) |
| { |
| eError = BridgeRIUpdateMEMDESCAddr(GetBridgeHandle(psImport->hDevConnection), |
| psMemDesc->hRIHandle, |
| psImport->sDeviceImport.sDevVAddr); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIUpdateMEMDESCAddr failed (Error=%d)", __func__, eError)); |
| } |
| } |
| } |
| #endif |
| |
| return PVRSRV_OK; |
| |
| failMap: |
| bDestroyed = _DevmemMemDescRelease(psMemDesc); |
| failCheck: |
| failParams: |
| if (!bDestroyed) |
| { |
| OSLockRelease(psMemDesc->sDeviceMemDesc.hLock); |
| } |
| PVR_ASSERT(eError != PVRSRV_OK); |
| failFlags: |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemMapToDeviceAddress(DEVMEM_MEMDESC *psMemDesc, |
| DEVMEM_HEAP *psHeap, |
| IMG_DEV_VIRTADDR sDevVirtAddr) |
| { |
| DEVMEM_IMPORT *psImport; |
| IMG_DEV_VIRTADDR sDevVAddr; |
| PVRSRV_ERROR eError; |
| IMG_BOOL bMap = IMG_TRUE; |
| IMG_BOOL bDestroyed = IMG_FALSE; |
| |
| /* Do not try to map unpinned memory */ |
| if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED) |
| { |
| eError = PVRSRV_ERROR_INVALID_MAP_REQUEST; |
| goto failFlags; |
| } |
| |
| OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock); |
| if (psHeap == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto failParams; |
| } |
| |
| if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0) |
| { |
| eError = PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED; |
| goto failCheck; |
| } |
| |
| /* Don't map memory for deferred allocations */ |
| if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC) |
| { |
| PVR_ASSERT(psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE); |
| bMap = IMG_FALSE; |
| } |
| |
| DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d", |
| __func__, |
| psMemDesc, |
| psMemDesc->sDeviceMemDesc.ui32RefCount, |
| psMemDesc->sDeviceMemDesc.ui32RefCount+1); |
| |
| psImport = psMemDesc->psImport; |
| _DevmemMemDescAcquire(psMemDesc); |
| |
| eError = _DevmemImportStructDevMap(psHeap, |
| bMap, |
| psImport, |
| sDevVirtAddr.uiAddr); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMap; |
| } |
| |
| sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr; |
| sDevVAddr.uiAddr += psMemDesc->uiOffset; |
| psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr; |
| psMemDesc->sDeviceMemDesc.ui32RefCount++; |
| |
| OSLockRelease(psMemDesc->sDeviceMemDesc.hLock); |
| |
| if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) |
| { |
| BridgeDevicememHistoryMap(GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->psImport->hPMR, |
| psMemDesc->uiOffset, |
| psMemDesc->sDeviceMemDesc.sDevVAddr, |
| psMemDesc->uiAllocSize, |
| psMemDesc->szText, |
| DevmemGetHeapLog2PageSize(psHeap), |
| psMemDesc->ui32AllocationIndex, |
| &psMemDesc->ui32AllocationIndex); |
| } |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI)) |
| { |
| if (psMemDesc->hRIHandle) |
| { |
| eError = BridgeRIUpdateMEMDESCAddr(GetBridgeHandle(psImport->hDevConnection), |
| psMemDesc->hRIHandle, |
| psImport->sDeviceImport.sDevVAddr); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIUpdateMEMDESCAddr failed (Error=%d)", __func__, eError)); |
| } |
| } |
| } |
| #endif |
| |
| return PVRSRV_OK; |
| |
| failMap: |
| bDestroyed = _DevmemMemDescRelease(psMemDesc); |
| failCheck: |
| failParams: |
| if (!bDestroyed) |
| { |
| OSLockRelease(psMemDesc->sDeviceMemDesc.hLock); |
| } |
| PVR_ASSERT(eError != PVRSRV_OK); |
| failFlags: |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemAcquireDevVirtAddr(DEVMEM_MEMDESC *psMemDesc, |
| IMG_DEV_VIRTADDR *psDevVirtAddr) |
| { |
| PVRSRV_ERROR eError; |
| |
| /* Do not try to map unpinned memory */ |
| if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED) |
| { |
| eError = PVRSRV_ERROR_INVALID_MAP_REQUEST; |
| goto failCheck; |
| } |
| |
| OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock); |
| DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d", |
| __func__, |
| psMemDesc, |
| psMemDesc->sDeviceMemDesc.ui32RefCount, |
| psMemDesc->sDeviceMemDesc.ui32RefCount+1); |
| |
| if (psMemDesc->sDeviceMemDesc.ui32RefCount == 0) |
| { |
| eError = PVRSRV_ERROR_DEVICEMEM_NO_MAPPING; |
| goto failRelease; |
| } |
| psMemDesc->sDeviceMemDesc.ui32RefCount++; |
| |
| *psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr; |
| OSLockRelease(psMemDesc->sDeviceMemDesc.hLock); |
| |
| return PVRSRV_OK; |
| |
| failRelease: |
| OSLockRelease(psMemDesc->sDeviceMemDesc.hLock); |
| PVR_ASSERT(eError != PVRSRV_OK); |
| failCheck: |
| return eError; |
| } |
| |
| IMG_INTERNAL void |
| DevmemReleaseDevVirtAddr(DEVMEM_MEMDESC *psMemDesc) |
| { |
| PVR_ASSERT(psMemDesc != NULL); |
| |
| OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock); |
| DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d", |
| __func__, |
| psMemDesc, |
| psMemDesc->sDeviceMemDesc.ui32RefCount, |
| psMemDesc->sDeviceMemDesc.ui32RefCount-1); |
| |
| PVR_ASSERT(psMemDesc->sDeviceMemDesc.ui32RefCount != 0); |
| |
| if (--psMemDesc->sDeviceMemDesc.ui32RefCount == 0) |
| { |
| if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) |
| { |
| BridgeDevicememHistoryUnmap(GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->psImport->hPMR, |
| psMemDesc->uiOffset, |
| psMemDesc->sDeviceMemDesc.sDevVAddr, |
| psMemDesc->uiAllocSize, |
| psMemDesc->szText, |
| DevmemGetHeapLog2PageSize(psMemDesc->psImport->sDeviceImport.psHeap), |
| psMemDesc->ui32AllocationIndex, |
| &psMemDesc->ui32AllocationIndex); |
| } |
| |
| _DevmemImportStructDevUnmap(psMemDesc->psImport); |
| OSLockRelease(psMemDesc->sDeviceMemDesc.hLock); |
| |
| _DevmemMemDescRelease(psMemDesc); |
| } |
| else |
| { |
| OSLockRelease(psMemDesc->sDeviceMemDesc.hLock); |
| } |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemAcquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc, |
| void **ppvCpuVirtAddr) |
| { |
| PVRSRV_ERROR eError; |
| |
| PVR_ASSERT(psMemDesc != NULL); |
| PVR_ASSERT(ppvCpuVirtAddr != NULL); |
| |
| if (psMemDesc->psImport->uiProperties & |
| (DEVMEM_PROPERTIES_UNPINNED | DEVMEM_PROPERTIES_SECURE)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Allocation is currently unpinned or a secure buffer. " |
| "Not possible to map to CPU!", |
| __func__)); |
| eError = PVRSRV_ERROR_INVALID_MAP_REQUEST; |
| goto failFlags; |
| } |
| |
| if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_NO_CPU_MAPPING) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: CPU Mapping is not possible on this allocation!", |
| __func__)); |
| eError = PVRSRV_ERROR_INVALID_MAP_REQUEST; |
| goto failFlags; |
| } |
| |
| OSLockAcquire(psMemDesc->sCPUMemDesc.hLock); |
| DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d", |
| __func__, |
| psMemDesc, |
| psMemDesc->sCPUMemDesc.ui32RefCount, |
| psMemDesc->sCPUMemDesc.ui32RefCount+1); |
| |
| if (psMemDesc->sCPUMemDesc.ui32RefCount++ == 0) |
| { |
| DEVMEM_IMPORT *psImport = psMemDesc->psImport; |
| IMG_UINT8 *pui8CPUVAddr; |
| |
| _DevmemMemDescAcquire(psMemDesc); |
| eError = _DevmemImportStructCPUMap(psImport); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMap; |
| } |
| |
| pui8CPUVAddr = psImport->sCPUImport.pvCPUVAddr; |
| pui8CPUVAddr += psMemDesc->uiOffset; |
| psMemDesc->sCPUMemDesc.pvCPUVAddr = pui8CPUVAddr; |
| } |
| *ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr; |
| |
| VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize); |
| |
| OSLockRelease(psMemDesc->sCPUMemDesc.hLock); |
| |
| return PVRSRV_OK; |
| |
| failMap: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| psMemDesc->sCPUMemDesc.ui32RefCount--; |
| |
| if (!_DevmemMemDescRelease(psMemDesc)) |
| { |
| OSLockRelease(psMemDesc->sCPUMemDesc.hLock); |
| } |
| failFlags: |
| return eError; |
| } |
| |
| IMG_INTERNAL void |
| DevmemReacquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc, |
| void **ppvCpuVirtAddr) |
| { |
| PVR_ASSERT(psMemDesc != NULL); |
| PVR_ASSERT(ppvCpuVirtAddr != NULL); |
| |
| if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_NO_CPU_MAPPING) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: CPU UnMapping is not possible on this allocation!", |
| __func__)); |
| return; |
| } |
| |
| OSLockAcquire(psMemDesc->sCPUMemDesc.hLock); |
| DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d", |
| __func__, |
| psMemDesc, |
| psMemDesc->sCPUMemDesc.ui32RefCount, |
| psMemDesc->sCPUMemDesc.ui32RefCount+1); |
| |
| *ppvCpuVirtAddr = NULL; |
| if (psMemDesc->sCPUMemDesc.ui32RefCount) |
| { |
| *ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr; |
| psMemDesc->sCPUMemDesc.ui32RefCount += 1; |
| } |
| |
| VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize); |
| OSLockRelease(psMemDesc->sCPUMemDesc.hLock); |
| } |
| |
| IMG_INTERNAL void |
| DevmemReleaseCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc) |
| { |
| PVR_ASSERT(psMemDesc != NULL); |
| |
| if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_NO_CPU_MAPPING) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: CPU UnMapping is not possible on this allocation!", |
| __func__)); |
| return; |
| } |
| |
| OSLockAcquire(psMemDesc->sCPUMemDesc.hLock); |
| DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d", |
| __func__, |
| psMemDesc, |
| psMemDesc->sCPUMemDesc.ui32RefCount, |
| psMemDesc->sCPUMemDesc.ui32RefCount-1); |
| |
| PVR_ASSERT(psMemDesc->sCPUMemDesc.ui32RefCount != 0); |
| |
| if (--psMemDesc->sCPUMemDesc.ui32RefCount == 0) |
| { |
| OSLockRelease(psMemDesc->sCPUMemDesc.hLock); |
| _DevmemImportStructCPUUnmap(psMemDesc->psImport); |
| _DevmemMemDescRelease(psMemDesc); |
| } |
| else |
| { |
| OSLockRelease(psMemDesc->sCPUMemDesc.hLock); |
| } |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemLocalGetImportHandle(DEVMEM_MEMDESC *psMemDesc, |
| IMG_HANDLE *phImport) |
| { |
| if ((psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE) == 0) |
| { |
| return PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION; |
| } |
| |
| *phImport = psMemDesc->psImport->hPMR; |
| |
| return PVRSRV_OK; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemGetImportUID(DEVMEM_MEMDESC *psMemDesc, |
| IMG_UINT64 *pui64UID) |
| { |
| DEVMEM_IMPORT *psImport = psMemDesc->psImport; |
| PVRSRV_ERROR eError; |
| |
| if (!(psImport->uiProperties & (DEVMEM_PROPERTIES_IMPORTED | |
| DEVMEM_PROPERTIES_EXPORTABLE))) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: This Memory (0x%p) doesn't support the functionality requested...", |
| __func__, psMemDesc)); |
| return PVRSRV_ERROR_INVALID_REQUEST; |
| } |
| |
| eError = BridgePMRGetUID(GetBridgeHandle(psImport->hDevConnection), |
| psImport->hPMR, |
| pui64UID); |
| |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemGetReservation(DEVMEM_MEMDESC *psMemDesc, |
| IMG_HANDLE *hReservation) |
| { |
| DEVMEM_IMPORT *psImport; |
| |
| PVR_ASSERT(psMemDesc); |
| psImport = psMemDesc->psImport; |
| |
| PVR_ASSERT(psImport); |
| *hReservation = psImport->sDeviceImport.hReservation; |
| |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| DevmemGetPMRData(DEVMEM_MEMDESC *psMemDesc, |
| IMG_HANDLE *phPMR, |
| IMG_DEVMEM_OFFSET_T *puiPMROffset) |
| { |
| DEVMEM_IMPORT *psImport; |
| |
| PVR_ASSERT(psMemDesc); |
| *puiPMROffset = psMemDesc->uiOffset; |
| psImport = psMemDesc->psImport; |
| |
| PVR_ASSERT(psImport); |
| *phPMR = psImport->hPMR; |
| |
| return PVRSRV_OK; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemGetFlags(DEVMEM_MEMDESC *psMemDesc, |
| DEVMEM_FLAGS_T *puiFlags) |
| { |
| DEVMEM_IMPORT *psImport; |
| |
| PVR_ASSERT(psMemDesc); |
| psImport = psMemDesc->psImport; |
| |
| PVR_ASSERT(psImport); |
| *puiFlags = psImport->uiFlags; |
| |
| return PVRSRV_OK; |
| } |
| |
| IMG_INTERNAL SHARED_DEV_CONNECTION |
| DevmemGetConnection(DEVMEM_MEMDESC *psMemDesc) |
| { |
| return psMemDesc->psImport->hDevConnection; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemLocalImport(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_HANDLE hExtHandle, |
| DEVMEM_FLAGS_T uiFlags, |
| DEVMEM_MEMDESC **ppsMemDescPtr, |
| IMG_DEVMEM_SIZE_T *puiSizePtr, |
| const IMG_CHAR *pszAnnotation) |
| { |
| DEVMEM_MEMDESC *psMemDesc = NULL; |
| DEVMEM_IMPORT *psImport; |
| IMG_DEVMEM_SIZE_T uiSize; |
| IMG_DEVMEM_ALIGN_T uiAlign; |
| IMG_HANDLE hPMR; |
| PVRSRV_ERROR eError; |
| |
| if (ppsMemDescPtr == NULL) |
| { |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto failParams; |
| } |
| |
| eError =_DevmemMemDescAlloc(&psMemDesc); |
| if (eError != PVRSRV_OK) |
| { |
| goto failMemDescAlloc; |
| } |
| |
| eError = _DevmemImportStructAlloc(hDevConnection, |
| &psImport); |
| if (eError != PVRSRV_OK) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto failImportAlloc; |
| } |
| |
| /* Get the PMR handle and its size from the server */ |
| eError = BridgePMRLocalImportPMR(GetBridgeHandle(hDevConnection), |
| hExtHandle, |
| &hPMR, |
| &uiSize, |
| &uiAlign); |
| if (eError != PVRSRV_OK) |
| { |
| goto failImport; |
| } |
| |
| _DevmemImportStructInit(psImport, |
| uiSize, |
| uiAlign, |
| uiFlags, |
| hPMR, |
| DEVMEM_PROPERTIES_IMPORTED | |
| DEVMEM_PROPERTIES_EXPORTABLE); |
| |
| _DevmemMemDescInit(psMemDesc, |
| 0, |
| psImport, |
| uiSize); |
| |
| *ppsMemDescPtr = psMemDesc; |
| if (puiSizePtr) |
| *puiSizePtr = uiSize; |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI)) |
| { |
| /* Attach RI information. |
| * Set backed size to 0 since this allocation has been allocated |
| * by the same process and has been accounted for. */ |
| eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection), |
| psMemDesc->psImport->hPMR, |
| sizeof("^"), |
| "^", |
| psMemDesc->uiOffset, |
| psMemDesc->psImport->uiSize, |
| IMG_TRUE, |
| IMG_FALSE, |
| &(psMemDesc->hRIHandle)); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (Error=%d)", __func__, eError)); |
| } |
| } |
| #endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| |
| |
| /* Copy the allocation descriptive name and size so it can be passed |
| * to DevicememHistory when the allocation gets mapped/unmapped |
| */ |
| _CheckAnnotationLength(pszAnnotation); |
| #if defined(__KERNEL__) |
| OSStringLCopy(psMemDesc->szText, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN); |
| #else |
| OSStringNCopy(psMemDesc->szText, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN); |
| psMemDesc->szText[DEVMEM_ANNOTATION_MAX_LEN - 1] = '\0'; |
| #endif /* if defined(__KERNEL__) */ |
| |
| return PVRSRV_OK; |
| |
| failImport: |
| _DevmemImportDiscard(psImport); |
| failImportAlloc: |
| _DevmemMemDescDiscard(psMemDesc); |
| failMemDescAlloc: |
| failParams: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemIsDevVirtAddrValid(DEVMEM_CONTEXT *psContext, |
| IMG_DEV_VIRTADDR sDevVAddr) |
| { |
| return BridgeDevmemIsVDevAddrValid(GetBridgeHandle(psContext->hDevConnection), |
| psContext->hDevMemServerContext, |
| sDevVAddr); |
| } |
| |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| DevmemGetFaultAddress(DEVMEM_CONTEXT *psContext, |
| IMG_DEV_VIRTADDR *psFaultAddress) |
| { |
| return BridgeDevmemGetFaultAddress(GetBridgeHandle(psContext->hDevConnection), |
| psContext->hDevMemServerContext, |
| psFaultAddress); |
| } |
| |
| IMG_INTERNAL IMG_UINT32 |
| DevmemGetHeapLog2PageSize(DEVMEM_HEAP *psHeap) |
| { |
| return psHeap->uiLog2Quantum; |
| } |
| |
| IMG_INTERNAL IMG_UINT32 |
| DevmemGetHeapTilingProperties(DEVMEM_HEAP *psHeap, |
| IMG_UINT32 *puiLog2ImportAlignment, |
| IMG_UINT32 *puiLog2TilingStrideFactor) |
| { |
| *puiLog2ImportAlignment = psHeap->uiLog2ImportAlignment; |
| *puiLog2TilingStrideFactor = psHeap->uiLog2TilingStrideFactor; |
| return PVRSRV_OK; |
| } |
| |
| /**************************************************************************/ /*! |
| @Function RegisterDevMemPFNotify |
| @Description Registers that the application wants to be signaled when a page |
| fault occurs. |
| |
| @Input psContext Memory context the process that would like to |
| be notified about. |
| @Input ui32PID The PID of the calling process. |
| @Input bRegister If true, register. If false, de-register. |
| @Return PVRSRV_ERROR: PVRSRV_OK on success. Otherwise, a PVRSRV_ |
| error code |
| */ /***************************************************************************/ |
| IMG_INTERNAL PVRSRV_ERROR |
| RegisterDevmemPFNotify(DEVMEM_CONTEXT *psContext, |
| IMG_UINT32 ui32PID, |
| IMG_BOOL bRegister) |
| { |
| PVRSRV_ERROR eError; |
| |
| eError = BridgeDevmemIntRegisterPFNotifyKM(GetBridgeHandle(psContext->hDevConnection), |
| psContext->hDevMemServerContext, |
| ui32PID, |
| bRegister); |
| if (eError == PVRSRV_ERROR_BRIDGE_CALL_FAILED) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Bridge Call Failed: This could suggest a UM/KM mismatch (%d)", |
| __func__, |
| (IMG_INT)(eError))); |
| } |
| |
| return eError; |
| } |
| |
| IMG_INTERNAL PVRSRV_ERROR |
| GetMaxDevMemSize(SHARED_DEV_CONNECTION hDevConnection, |
| IMG_DEVMEM_SIZE_T *puiLMASize, |
| IMG_DEVMEM_SIZE_T *puiUMASize) |
| { |
| return BridgeGetMaxDevMemSize(GetBridgeHandle(hDevConnection), |
| puiLMASize, |
| puiUMASize); |
| } |