| /*************************************************************************/ /*! |
| @File |
| @Title Physmem (PMR) abstraction |
| @Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved |
| @Description Part of the memory management. This module is responsible for |
| the "PMR" abstraction. A PMR (Physical Memory Resource) |
| represents some unit of physical memory which is |
| allocated/freed/mapped/unmapped as an indivisible unit |
| (higher software levels provide an abstraction above that |
| to deal with dividing this down into smaller manageable units). |
| Importantly, this module knows nothing of virtual memory, or |
| of MMUs etc., with one excusable exception. We have the |
| concept of a "page size", which really means nothing in |
| physical memory, but represents a "contiguity quantum" such |
| that the higher level modules which map this memory are able |
| to verify that it matches the needs of the page size for the |
| virtual realm into which it is being mapped. |
| @License Dual MIT/GPLv2 |
| |
| The contents of this file are subject to the MIT license as set out below. |
| |
| Permission is hereby granted, free of charge, to any person obtaining a copy |
| of this software and associated documentation files (the "Software"), to deal |
| in the Software without restriction, including without limitation the rights |
| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| copies of the Software, and to permit persons to whom the Software is |
| furnished to do so, subject to the following conditions: |
| |
| The above copyright notice and this permission notice shall be included in |
| all copies or substantial portions of the Software. |
| |
| Alternatively, the contents of this file may be used under the terms of |
| the GNU General Public License Version 2 ("GPL") in which case the provisions |
| of GPL are applicable instead of those above. |
| |
| If you wish to allow use of your version of this file only under the terms of |
| GPL, and not to allow others to use your version of this file under the terms |
| of the MIT license, indicate your decision by deleting the provisions above |
| and replace them with the notice and other provisions required by GPL as set |
| out in the file called "GPL-COPYING" included in this distribution. If you do |
| not delete the provisions above, a recipient may use your version of this file |
| under the terms of either the MIT license or GPL. |
| |
| This License is also included in this distribution in the file called |
| "MIT-COPYING". |
| |
| EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS |
| PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING |
| BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR |
| PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR |
| COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER |
| IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| */ /**************************************************************************/ |
| |
| #include "img_types.h" |
| #include "img_defs.h" |
| #include "pvr_debug.h" |
| #include "pvrsrv_error.h" |
| |
| #include "pdump.h" |
| #include "devicemem_server_utils.h" |
| |
| #include "osfunc.h" |
| #include "pdump_km.h" |
| #include "pdump_physmem.h" |
| #include "pmr_impl.h" |
| #include "pmr_os.h" |
| #include "pvrsrv.h" |
| |
| #include "allocmem.h" |
| #include "lock.h" |
| |
| #if defined(SUPPORT_SECURE_EXPORT) |
| #include "secure_export.h" |
| #include "ossecure_export.h" |
| #endif |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| #include "ri_server.h" |
| #endif |
| |
| /* ourselves */ |
| #include "pmr.h" |
| |
| #if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS) |
| #include "mmap_stats.h" |
| #endif |
| |
| /* A "context" for the physical memory block resource allocator. |
| |
| Context is probably the wrong word. |
| |
| There is almost certainly only one of these, ever, in the system. |
| But, let's keep the notion of a context anyway, "just-in-case". |
| */ |
| static struct _PMR_CTX_ |
| { |
| /* For debugging, and PDump, etc., let's issue a forever |
| incrementing serial number to each allocation. */ |
| IMG_UINT64 uiNextSerialNum; |
| |
| /* For security, we only allow a PMR to be mapped if the caller |
| knows its key. We can pseudo-randomly generate keys */ |
| IMG_UINT64 uiNextKey; |
| |
| /* For debugging only, I guess: Number of live PMRs */ |
| IMG_UINT32 uiNumLivePMRs; |
| |
| /* Lock for this structure */ |
| POS_LOCK hLock; |
| |
| /* In order to seed the uiNextKey, we enforce initialisation at |
| driver load time. Also, we can debug check at driver unload |
| that the PMR count is zero. */ |
| IMG_BOOL bModuleInitialised; |
| } _gsSingletonPMRContext = { 1, 0, 0, NULL, IMG_FALSE }; |
| |
| |
| /* A PMR. One per physical allocation. May be "shared". |
| |
| "shared" is ambiguous. We need to be careful with terminology. |
| There are two ways in which a PMR may be "shared" and we need to be |
| sure that we are clear which we mean. |
| |
| i) multiple small allocations living together inside one PMR; |
| |
| ii) one single allocation filling a PMR but mapped into multiple |
| memory contexts. |
| |
| This is more important further up the stack - at this level, all we |
| care is that the PMR is being referenced multiple times. |
| */ |
| struct _PMR_ |
| { |
| /* This object is strictly refcounted. References include: |
| - mapping |
| - live handles (to this object) |
| - live export handles |
| (thus it is normal for allocated and exported memory to have a refcount of 3) |
| The object is destroyed when and only when the refcount reaches 0 |
| */ |
| |
| /* Device node on which this PMR was created and is valid */ |
| PVRSRV_DEVICE_NODE *psDevNode; |
| |
| /* |
| Physical address translation (device <> cpu) is done on a per device |
| basis which means we need the physical heap info |
| */ |
| PHYS_HEAP *psPhysHeap; |
| |
| ATOMIC_T iRefCount; |
| |
| /* lock count - this is the number of times |
| PMRLockSysPhysAddresses() has been called, less the number of |
| PMRUnlockSysPhysAddresses() calls. This is arguably here for |
| debug reasons only, as the refcount is already incremented as a |
| matter of course. Really, this just allows us to trap protocol |
| errors: i.e. calling PMRSysPhysAddr(), |
| without a lock, or calling PMRUnlockSysPhysAddresses() too many |
| or too few times. */ |
| ATOMIC_T iLockCount; |
| |
| /* Lock for this structure */ |
| POS_LOCK hLock; |
| |
| /* Incrementing serial number to each allocation. */ |
| IMG_UINT64 uiSerialNum; |
| |
| /* For security, we only allow a PMR to be mapped if the caller |
| knows its key. We can pseudo-randomly generate keys */ |
| PMR_PASSWORD_T uiKey; |
| |
| /* Callbacks for per-flavour functions */ |
| const PMR_IMPL_FUNCTAB *psFuncTab; |
| |
| /* Data associated with the "subtype" */ |
| PMR_IMPL_PRIVDATA pvFlavourData; |
| |
| /* What kind of PMR do we have? */ |
| PMR_IMPL_TYPE eFlavour; |
| |
| /* And for pdump */ |
| const IMG_CHAR *pszPDumpDefaultMemspaceName; |
| |
| /* Allocation annotation */ |
| IMG_CHAR szAnnotation[DEVMEM_ANNOTATION_MAX_LEN]; |
| |
| #if defined(PDUMP) |
| |
| IMG_HANDLE hPDumpAllocHandle; |
| |
| IMG_UINT32 uiNumPDumpBlocks; |
| #endif |
| |
| /* Logical size of allocation. "logical", because a PMR can |
| represent memory that will never physically exist. This is the |
| amount of virtual space that the PMR would consume when it's |
| mapped into a virtual allocation. */ |
| PMR_SIZE_T uiLogicalSize; |
| |
| /* Mapping table for the allocation. |
| PMR's can be sparse in which case not all the "logic" addresses |
| in it are valid. We need to know which addresses are and aren't |
| valid when mapping or reading the PMR. |
| The mapping table translates "logical" offsets into physical |
| offsets which is what we always pass to the PMR factory |
| (so it doesn't have to be concerned about sparseness issues) */ |
| PMR_MAPPING_TABLE *psMappingTable; |
| |
| /* Indicates whether this PMR has been allocated as sparse. |
| * The condition for this variable to be set at allocation time is: |
| * (numVirtChunks != numPhysChunks) || (numVirtChunks > 1) |
| */ |
| IMG_BOOL bSparseAlloc; |
| |
| /* Indicates whether this PMR has been unpinned. |
| * By default, all PMRs are pinned at creation. |
| */ |
| IMG_BOOL bIsUnpinned; |
| |
| /* Minimum Physical Contiguity Guarantee. Might be called "page |
| size", but that would be incorrect, as page size is something |
| meaningful only in virtual realm. This contiguity guarantee |
| provides an inequality that can be verified/asserted/whatever |
| to ensure that this PMR conforms to the page size requirement |
| of the place the PMR gets mapped. (May be used to select an |
| appropriate heap in variable page size systems) |
| |
| The absolutely necessary condition is this: |
| |
| device MMU page size <= actual physical contiguity. |
| |
| We go one step further in order to be able to provide an early warning / early compatibility check and say this: |
| |
| device MMU page size <= 2**(uiLog2ContiguityGuarantee) <= actual physical contiguity. |
| |
| In this way, it is possible to make the page table reservation |
| in the device MMU without even knowing the granularity of the |
| physical memory (i.e. useful for being able to allocate virtual |
| before physical) |
| */ |
| PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee; |
| |
| /* Flags. We store a copy of the "PMR flags" (usually a subset of |
| the flags given at allocation time) and return them to any |
| caller of PMR_Flags(). The intention of these flags is that |
| the ones stored here are used to represent permissions, such |
| that no one is able to map a PMR in a mode in which they are not |
| allowed, e.g. writeable for a read-only PMR, etc. */ |
| PMR_FLAGS_T uiFlags; |
| |
| /* Do we really need this? For now we'll keep it, until we know we don't. */ |
| /* NB: this is not the "memory context" in client terms - this is |
| _purely_ the "PMR" context, of which there is almost certainly only |
| ever one per system as a whole, but we'll keep the concept |
| anyway, just-in-case. */ |
| struct _PMR_CTX_ *psContext; |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| /* |
| * Stored handle to PMR RI entry |
| */ |
| void *hRIHandle; |
| #endif |
| }; |
| |
| /* do we need a struct for the export handle? I'll use one for now, but if nothing goes in it, we'll lose it */ |
| struct _PMR_EXPORT_ |
| { |
| struct _PMR_ *psPMR; |
| }; |
| |
| struct _PMR_PAGELIST_ |
| { |
| struct _PMR_ *psReferencePMR; |
| }; |
| |
| PPVRSRV_DEVICE_NODE PMRGetExportDeviceNode(PMR_EXPORT *psExportPMR) |
| { |
| PPVRSRV_DEVICE_NODE psReturnedDeviceNode = NULL; |
| |
| PVR_ASSERT(psExportPMR != NULL); |
| if (psExportPMR) |
| { |
| PVR_ASSERT(psExportPMR->psPMR != NULL); |
| if (psExportPMR->psPMR) |
| { |
| PVR_ASSERT(OSAtomicRead(&psExportPMR->psPMR->iRefCount) > 0); |
| if (OSAtomicRead(&psExportPMR->psPMR->iRefCount) > 0) |
| { |
| psReturnedDeviceNode = PMR_DeviceNode(psExportPMR->psPMR); |
| } |
| } |
| } |
| |
| return psReturnedDeviceNode; |
| } |
| |
| static PVRSRV_ERROR |
| _PMRCreate(PMR_SIZE_T uiLogicalSize, |
| PMR_SIZE_T uiChunkSize, |
| IMG_UINT32 ui32NumPhysChunks, |
| IMG_UINT32 ui32NumVirtChunks, |
| IMG_UINT32 *pui32MappingTable, |
| PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee, |
| PMR_FLAGS_T uiFlags, |
| PMR **ppsPMR) |
| { |
| void *pvPMRLinAddr; |
| PMR *psPMR; |
| PMR_MAPPING_TABLE *psMappingTable; |
| struct _PMR_CTX_ *psContext; |
| IMG_UINT32 i,ui32Temp=0; |
| IMG_UINT32 ui32Remainder; |
| PVRSRV_ERROR eError; |
| IMG_BOOL bSparse = IMG_FALSE; |
| |
| psContext = &_gsSingletonPMRContext; |
| |
| /* Do we have a sparse allocation? */ |
| if ( (ui32NumVirtChunks != ui32NumPhysChunks) || |
| (ui32NumVirtChunks > 1) ) |
| { |
| bSparse = IMG_TRUE; |
| } |
| |
| /* Extra checks required for sparse PMRs */ |
| if (uiLogicalSize != uiChunkSize) |
| { |
| /* Check the logical size and chunk information agree with each other */ |
| if (uiLogicalSize != (uiChunkSize * ui32NumVirtChunks)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Bad mapping size (uiLogicalSize = 0x%llx, uiChunkSize = 0x%llx, ui32NumVirtChunks = %d)", |
| __func__, (unsigned long long)uiLogicalSize, (unsigned long long)uiChunkSize, ui32NumVirtChunks)); |
| return PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE; |
| } |
| |
| /* Check that the chunk size is a multiple of the contiguity */ |
| OSDivide64(uiChunkSize, (1<< uiLog2ContiguityGuarantee), &ui32Remainder); |
| if (ui32Remainder) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Bad chunk size, must be a multiple of the contiguity " |
| "(uiChunkSize = 0x%llx, uiLog2ContiguityGuarantee = %u)", |
| __func__, |
| (unsigned long long) uiChunkSize, |
| uiLog2ContiguityGuarantee)); |
| return PVRSRV_ERROR_PMR_BAD_CHUNK_SIZE; |
| } |
| } |
| |
| pvPMRLinAddr = OSAllocMem(sizeof(*psPMR) + sizeof(*psMappingTable) + sizeof(IMG_UINT32) * ui32NumVirtChunks); |
| |
| if (pvPMRLinAddr == NULL) |
| { |
| return PVRSRV_ERROR_OUT_OF_MEMORY; |
| } |
| |
| psPMR = (PMR *) pvPMRLinAddr; |
| psMappingTable = (PMR_MAPPING_TABLE *) (((IMG_CHAR *) pvPMRLinAddr) + sizeof(*psPMR)); |
| |
| eError = OSLockCreate(&psPMR->hLock); |
| if (eError != PVRSRV_OK) |
| { |
| OSFreeMem(psPMR); |
| return eError; |
| } |
| |
| /* Setup the mapping table */ |
| psMappingTable->uiChunkSize = uiChunkSize; |
| psMappingTable->ui32NumVirtChunks = ui32NumVirtChunks; |
| psMappingTable->ui32NumPhysChunks = ui32NumPhysChunks; |
| OSCachedMemSet(&psMappingTable->aui32Translation[0], 0xFF, sizeof(psMappingTable->aui32Translation[0])* |
| ui32NumVirtChunks); |
| for (i=0; i<ui32NumPhysChunks; i++) |
| { |
| ui32Temp = pui32MappingTable[i]; |
| psMappingTable->aui32Translation[ui32Temp] = ui32Temp; |
| } |
| |
| /* Setup the PMR */ |
| OSAtomicWrite(&psPMR->iRefCount, 0); |
| |
| /* If allocation is not made on demand, it will be backed now and |
| * backing will not be removed until the PMR is destroyed, therefore |
| * we can initialise the iLockCount to 1 rather than 0. |
| */ |
| OSAtomicWrite(&psPMR->iLockCount, (PVRSRV_CHECK_ON_DEMAND(uiFlags) ? 0 : 1)); |
| |
| psPMR->psContext = psContext; |
| psPMR->uiLogicalSize = uiLogicalSize; |
| psPMR->uiLog2ContiguityGuarantee = uiLog2ContiguityGuarantee; |
| psPMR->uiFlags = uiFlags; |
| psPMR->psMappingTable = psMappingTable; |
| psPMR->bSparseAlloc = bSparse; |
| psPMR->bIsUnpinned = IMG_FALSE; |
| psPMR->szAnnotation[0] = '\0'; |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| psPMR->hRIHandle = NULL; |
| #endif |
| |
| OSLockAcquire(psContext->hLock); |
| psPMR->uiKey = psContext->uiNextKey; |
| psPMR->uiSerialNum = psContext->uiNextSerialNum; |
| psContext->uiNextKey = (0x80200003 * psContext->uiNextKey) |
| ^ (0xf00f0081 * (uintptr_t)pvPMRLinAddr); |
| psContext->uiNextSerialNum ++; |
| *ppsPMR = psPMR; |
| PVR_DPF((PVR_DBG_MESSAGE, "pmr.c: created PMR @0x%p", psPMR)); |
| /* Increment live PMR count */ |
| psContext->uiNumLivePMRs ++; |
| OSLockRelease(psContext->hLock); |
| |
| return PVRSRV_OK; |
| } |
| |
| /* This function returns true if the PMR is in use and false otherwise. |
| * This function is not thread safe and hence the caller |
| * needs to ensure the thread safety by explicitly taking |
| * the lock on the PMR or through other means */ |
| IMG_BOOL PMRIsPMRLive(PMR *psPMR) |
| { |
| return (OSAtomicRead(&psPMR->iRefCount) > 0); |
| } |
| |
| static IMG_UINT32 |
| _Ref(PMR *psPMR) |
| { |
| PVR_ASSERT(OSAtomicRead(&psPMR->iRefCount) >= 0); |
| /* We need to ensure that this function is always executed under |
| * PMRLock. The only exception acceptable is the unloading of the driver. |
| */ |
| return OSAtomicIncrement(&psPMR->iRefCount); |
| } |
| |
| static IMG_UINT32 |
| _Unref(PMR *psPMR) |
| { |
| PVR_ASSERT(OSAtomicRead(&psPMR->iRefCount) > 0); |
| /* We need to ensure that this function is always executed under |
| * PMRLock. The only exception acceptable is the unloading of the driver. |
| */ |
| return OSAtomicDecrement(&psPMR->iRefCount); |
| } |
| |
| static void |
| _UnrefAndMaybeDestroy(PMR *psPMR) |
| { |
| PVRSRV_ERROR eError2; |
| struct _PMR_CTX_ *psCtx; |
| IMG_INT iRefCount; |
| |
| PVR_ASSERT(psPMR != NULL); |
| |
| /* Acquire PMR factory lock if provided */ |
| if(psPMR->psFuncTab->pfnGetPMRFactoryLock) |
| { |
| psPMR->psFuncTab->pfnGetPMRFactoryLock(); |
| } |
| |
| iRefCount = _Unref(psPMR); |
| |
| if (iRefCount == 0) |
| { |
| if (psPMR->psFuncTab->pfnFinalize != NULL) |
| { |
| eError2 = psPMR->psFuncTab->pfnFinalize(psPMR->pvFlavourData); |
| |
| /* PMR unref can be called asynchronously by the kernel or other |
| * third party modules (eg. display) which doesn't go through the |
| * usual services bridge. The same PMR can be referenced simultaneously |
| * in a different path that results in a race condition. |
| * Hence depending on the race condition, a factory may refuse to destroy |
| * the resource associated with this PMR if a reference on it was taken |
| * prior to unref. In that case the PMR factory function returns the error. |
| * |
| * When such an error is encountered, the factory needs to ensure the state |
| * associated with PMR is undisturbed. At this point we just bail out from |
| * freeing the PMR itself. The PMR handle will then be freed at a later point |
| * when the same PMR is unreferenced. |
| * */ |
| if (PVRSRV_ERROR_PMR_STILL_REFERENCED == eError2) |
| { |
| if(psPMR->psFuncTab->pfnReleasePMRFactoryLock) |
| { |
| psPMR->psFuncTab->pfnReleasePMRFactoryLock(); |
| } |
| return; |
| } |
| PVR_ASSERT (eError2 == PVRSRV_OK); /* can we do better? */ |
| } |
| #if defined(PDUMP) |
| PDumpPMRFreePMR(psPMR, |
| psPMR->uiLogicalSize, |
| (1 << psPMR->uiLog2ContiguityGuarantee), |
| psPMR->uiLog2ContiguityGuarantee, |
| psPMR->hPDumpAllocHandle); |
| #endif |
| |
| #if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS) |
| /* This PMR is about to be destroyed, update its mmap stats record (if present) |
| * to avoid dangling pointer. Additionally, this is required because mmap stats |
| * are identified by PMRs and a new PMR down the line "might" get the same address |
| * as the one we're about to free and we'd like 2 different entries in mmaps |
| * stats for such cases */ |
| MMapStatsRemovePMR(psPMR); |
| #endif |
| |
| #ifdef PVRSRV_NEED_PVR_ASSERT |
| /* If not backed on demand, iLockCount should be 1 otherwise it should be 0 */ |
| PVR_ASSERT(OSAtomicRead(&psPMR->iLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1)); |
| #endif |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| { |
| PVRSRV_ERROR eError; |
| |
| /* Delete RI entry */ |
| if (psPMR->hRIHandle) |
| { |
| eError = RIDeletePMREntryKM (psPMR->hRIHandle); |
| |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: RIDeletePMREntryKM failed: %s", |
| __func__, |
| PVRSRVGetErrorString(eError))); |
| /* continue destroying the PMR */ |
| } |
| } |
| } |
| #endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */ |
| psCtx = psPMR->psContext; |
| |
| OSLockDestroy(psPMR->hLock); |
| |
| /* Release PMR factory lock acquired if any */ |
| if(psPMR->psFuncTab->pfnReleasePMRFactoryLock) |
| { |
| psPMR->psFuncTab->pfnReleasePMRFactoryLock(); |
| } |
| |
| OSFreeMem(psPMR); |
| |
| /* Decrement live PMR count. Probably only of interest for debugging */ |
| PVR_ASSERT(psCtx->uiNumLivePMRs > 0); |
| |
| OSLockAcquire(psCtx->hLock); |
| psCtx->uiNumLivePMRs --; |
| OSLockRelease(psCtx->hLock); |
| } |
| else |
| { |
| /* Release PMR factory lock acquired if any */ |
| if(psPMR->psFuncTab->pfnReleasePMRFactoryLock) |
| { |
| psPMR->psFuncTab->pfnReleasePMRFactoryLock(); |
| } |
| } |
| } |
| |
| static IMG_BOOL _PMRIsSparse(const PMR *psPMR) |
| { |
| return psPMR->bSparseAlloc; |
| } |
| |
| PVRSRV_ERROR |
| PMRCreatePMR(PVRSRV_DEVICE_NODE *psDevNode, |
| PHYS_HEAP *psPhysHeap, |
| PMR_SIZE_T uiLogicalSize, |
| PMR_SIZE_T uiChunkSize, |
| IMG_UINT32 ui32NumPhysChunks, |
| IMG_UINT32 ui32NumVirtChunks, |
| IMG_UINT32 *pui32MappingTable, |
| PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee, |
| PMR_FLAGS_T uiFlags, |
| const IMG_CHAR *pszAnnotation, |
| const PMR_IMPL_FUNCTAB *psFuncTab, |
| PMR_IMPL_PRIVDATA pvPrivData, |
| PMR_IMPL_TYPE eType, |
| PMR **ppsPMRPtr, |
| IMG_UINT32 ui32PDumpFlags) |
| { |
| PMR *psPMR = NULL; |
| PVRSRV_ERROR eError; |
| |
| eError = _PMRCreate(uiLogicalSize, |
| uiChunkSize, |
| ui32NumPhysChunks, |
| ui32NumVirtChunks, |
| pui32MappingTable, |
| uiLog2ContiguityGuarantee, |
| uiFlags, |
| &psPMR); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| psPMR->psDevNode = psDevNode; |
| psPMR->psPhysHeap = psPhysHeap; |
| psPMR->psFuncTab = psFuncTab; |
| psPMR->pszPDumpDefaultMemspaceName = PhysHeapPDumpMemspaceName(psPhysHeap); |
| psPMR->pvFlavourData = pvPrivData; |
| psPMR->eFlavour = eType; |
| OSAtomicWrite(&psPMR->iRefCount, 1); |
| |
| OSStringLCopy(psPMR->szAnnotation, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN); |
| |
| #if defined(PDUMP) |
| { |
| PMR_FLAGS_T uiFlags = psPMR->uiFlags; |
| IMG_BOOL bInitialise = IMG_FALSE; |
| IMG_UINT32 ui32InitValue = 0; |
| |
| if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags)) |
| { |
| bInitialise = IMG_TRUE; |
| } |
| else if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags)) |
| { |
| ui32InitValue = 0xDEADBEEF; |
| bInitialise = IMG_TRUE; |
| } |
| |
| PDumpPMRMallocPMR(psPMR, |
| (uiChunkSize * ui32NumVirtChunks), |
| 1ULL<<uiLog2ContiguityGuarantee, |
| uiChunkSize, |
| ui32NumPhysChunks, |
| ui32NumVirtChunks, |
| pui32MappingTable, |
| uiLog2ContiguityGuarantee, |
| bInitialise, |
| ui32InitValue, |
| &psPMR->hPDumpAllocHandle, |
| ui32PDumpFlags); |
| } |
| #endif |
| |
| *ppsPMRPtr = psPMR; |
| |
| return PVRSRV_OK; |
| |
| /* |
| * error exit paths follow |
| */ |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR PMRLockSysPhysAddressesNested(PMR *psPMR, |
| IMG_UINT32 ui32NestingLevel) |
| { |
| PVRSRV_ERROR eError; |
| |
| PVR_ASSERT(psPMR != NULL); |
| |
| /* Note: taking this lock is not required to protect the PMR reference count, |
| * because the PMR reference count is atomic. |
| * Rather, taking the lock here guarantees that no caller will exit this function |
| * without the underlying physical addresses being locked. |
| */ |
| OSLockAcquireNested(psPMR->hLock, ui32NestingLevel); |
| /* We also count the locks as references, so that the PMR is not |
| freed while someone is using a physical address. */ |
| /* "lock" here simply means incrementing the refcount. It means |
| the refcount is multipurpose, but that's okay. We only have to |
| promise that physical addresses are valid after this point, and |
| remain valid until the corresponding |
| PMRUnlockSysPhysAddressesOSMem() */ |
| _Ref(psPMR); |
| |
| /* Also count locks separately from other types of references, to |
| allow for debug assertions */ |
| |
| /* Only call callback if lockcount transitions from 0 to 1 (or 1 to 2 if not backed on demand) */ |
| if (OSAtomicIncrement(&psPMR->iLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 1 : 2)) |
| { |
| if (psPMR->psFuncTab->pfnLockPhysAddresses != NULL) |
| { |
| /* must always have lock and unlock in pairs! */ |
| PVR_ASSERT(psPMR->psFuncTab->pfnUnlockPhysAddresses != NULL); |
| |
| eError = psPMR->psFuncTab->pfnLockPhysAddresses(psPMR->pvFlavourData); |
| |
| if (eError != PVRSRV_OK) |
| { |
| goto e1; |
| } |
| } |
| } |
| OSLockRelease(psPMR->hLock); |
| |
| return PVRSRV_OK; |
| |
| e1: |
| OSAtomicDecrement(&psPMR->iLockCount); |
| _Unref(psPMR); |
| PVR_ASSERT(OSAtomicRead(&psPMR->iRefCount) != 0); |
| OSLockRelease(psPMR->hLock); |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRLockSysPhysAddresses(PMR *psPMR) |
| { |
| return PMRLockSysPhysAddressesNested(psPMR, 0); |
| } |
| |
| PVRSRV_ERROR |
| PMRUnlockSysPhysAddresses(PMR *psPMR) |
| { |
| return PMRUnlockSysPhysAddressesNested(psPMR, 2); |
| } |
| |
| PVRSRV_ERROR |
| PMRUnlockSysPhysAddressesNested(PMR *psPMR, IMG_UINT32 ui32NestingLevel) |
| { |
| PVRSRV_ERROR eError; |
| |
| PVR_ASSERT(psPMR != NULL); |
| |
| /* Acquiring the lock here, as well as during the Lock operation ensures |
| * the lock count hitting zero and the unlocking of the phys addresses is |
| * an atomic operation |
| */ |
| OSLockAcquireNested(psPMR->hLock, ui32NestingLevel); |
| PVR_ASSERT(OSAtomicRead(&psPMR->iLockCount) > (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1)); |
| |
| if (OSAtomicDecrement(&psPMR->iLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1)) |
| { |
| if (psPMR->psFuncTab->pfnUnlockPhysAddresses != NULL) |
| { |
| PVR_ASSERT(psPMR->psFuncTab->pfnLockPhysAddresses != NULL); |
| |
| eError = psPMR->psFuncTab->pfnUnlockPhysAddresses(psPMR->pvFlavourData); |
| /* must never fail */ |
| PVR_ASSERT(eError == PVRSRV_OK); |
| } |
| } |
| |
| OSLockRelease(psPMR->hLock); |
| |
| /* We also count the locks as references, so that the PMR is not |
| freed while someone is using a physical address. */ |
| _UnrefAndMaybeDestroy(psPMR); |
| |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| PMRUnpinPMR(PMR *psPMR, IMG_BOOL bDevMapped) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| |
| PVR_ASSERT(psPMR != NULL); |
| |
| OSLockAcquire(psPMR->hLock); |
| /* Stop if we still have references on the PMR */ |
| if ( ( bDevMapped && (OSAtomicRead(&psPMR->iRefCount) > 2)) |
| || (!bDevMapped && (OSAtomicRead(&psPMR->iRefCount) > 1)) ) |
| { |
| OSLockRelease(psPMR->hLock); |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: PMR is still referenced %u times. " |
| "That means this PMR is probably exported or used somewhere else. " |
| "Allowed are 2 references if it is mapped to device, otherwise 1.", |
| __func__, |
| OSAtomicRead(&psPMR->iRefCount))); |
| |
| eError = PVRSRV_ERROR_PMR_STILL_REFERENCED; |
| goto e_exit; |
| } |
| OSLockRelease(psPMR->hLock); |
| |
| if (psPMR->psFuncTab->pfnUnpinMem != NULL) |
| { |
| eError = psPMR->psFuncTab->pfnUnpinMem(psPMR->pvFlavourData); |
| if (eError == PVRSRV_OK) |
| { |
| psPMR->bIsUnpinned = IMG_TRUE; |
| } |
| } |
| |
| e_exit: |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRPinPMR(PMR *psPMR) |
| { |
| PVRSRV_ERROR eError= PVRSRV_OK; |
| |
| PVR_ASSERT(psPMR != NULL); |
| |
| if (psPMR->psFuncTab->pfnPinMem != NULL) |
| { |
| eError = psPMR->psFuncTab->pfnPinMem(psPMR->pvFlavourData, |
| psPMR->psMappingTable); |
| if (eError == PVRSRV_OK) |
| { |
| psPMR->bIsUnpinned = IMG_FALSE; |
| } |
| } |
| |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRMakeLocalImportHandle(PMR *psPMR, |
| PMR **ppsPMR) |
| { |
| PMRRefPMR(psPMR); |
| *ppsPMR = psPMR; |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| PMRUnmakeLocalImportHandle(PMR *psPMR) |
| { |
| PMRUnrefPMR(psPMR); |
| return PVRSRV_OK; |
| } |
| |
| /* |
| Note: |
| We pass back the PMR as it was passed in as a different handle type |
| (DEVMEM_MEM_IMPORT) and it allows us to change the import structure |
| type if we should need to embed any meta data in it. |
| */ |
| PVRSRV_ERROR |
| PMRLocalImportPMR(PMR *psPMR, |
| PMR **ppsPMR, |
| IMG_DEVMEM_SIZE_T *puiSize, |
| IMG_DEVMEM_ALIGN_T *puiAlign) |
| { |
| _Ref(psPMR); |
| |
| /* Return the PMR */ |
| *ppsPMR = psPMR; |
| *puiSize = psPMR->uiLogicalSize; |
| *puiAlign = 1ULL << psPMR->uiLog2ContiguityGuarantee; |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| PMRGetUID(PMR *psPMR, |
| IMG_UINT64 *pui64UID) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| |
| *pui64UID = psPMR->uiSerialNum; |
| |
| return PVRSRV_OK; |
| } |
| |
| #if defined(SUPPORT_INSECURE_EXPORT) |
| PVRSRV_ERROR |
| PMRExportPMR(PMR *psPMR, |
| PMR_EXPORT **ppsPMRExportPtr, |
| PMR_SIZE_T *puiSize, |
| PMR_LOG2ALIGN_T *puiLog2Contig, |
| PMR_PASSWORD_T *puiPassword) |
| { |
| IMG_UINT64 uiPassword; |
| PMR_EXPORT *psPMRExport; |
| |
| uiPassword = psPMR->uiKey; |
| |
| psPMRExport = OSAllocMem(sizeof(*psPMRExport)); |
| if (psPMRExport == NULL) |
| { |
| return PVRSRV_ERROR_OUT_OF_MEMORY; |
| } |
| |
| psPMRExport->psPMR = psPMR; |
| _Ref(psPMR); |
| |
| *ppsPMRExportPtr = psPMRExport; |
| *puiSize = psPMR->uiLogicalSize; |
| *puiLog2Contig = psPMR->uiLog2ContiguityGuarantee; |
| *puiPassword = uiPassword; |
| |
| return PVRSRV_OK; |
| } |
| |
| |
| PVRSRV_ERROR |
| PMRUnexportPMR(PMR_EXPORT *psPMRExport) |
| { |
| PVR_ASSERT(psPMRExport != NULL); |
| PVR_ASSERT(psPMRExport->psPMR != NULL); |
| PVR_ASSERT(OSAtomicRead(&psPMRExport->psPMR->iRefCount) > 0); |
| |
| _UnrefAndMaybeDestroy(psPMRExport->psPMR); |
| |
| OSFreeMem(psPMRExport); |
| |
| return PVRSRV_OK; |
| } |
| |
| |
| PVRSRV_ERROR |
| PMRImportPMR(PMR_EXPORT *psPMRExport, |
| PMR_PASSWORD_T uiPassword, |
| PMR_SIZE_T uiSize, |
| PMR_LOG2ALIGN_T uiLog2Contig, |
| PMR **ppsPMR) |
| { |
| PMR *psPMR; |
| |
| PVR_ASSERT(OSAtomicRead(&psPMRExport->psPMR->iRefCount) > 0); |
| |
| psPMR = psPMRExport->psPMR; |
| |
| |
| if (psPMR->uiKey != uiPassword) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "PMRImport: Import failed, password specified does not match the export")); |
| return PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR; |
| } |
| |
| if (psPMR->uiLogicalSize != uiSize || psPMR->uiLog2ContiguityGuarantee != uiLog2Contig) |
| { |
| return PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES; |
| } |
| |
| _Ref(psPMR); |
| |
| *ppsPMR = psPMR; |
| |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| PMRUnimportPMR(PMR *psPMR) |
| { |
| _UnrefAndMaybeDestroy(psPMR); |
| |
| return PVRSRV_OK; |
| } |
| |
| #else /* if defined(SUPPORT_INSECURE_EXPORT) */ |
| |
| PVRSRV_ERROR |
| PMRExportPMR(PMR *psPMR, |
| PMR_EXPORT **ppsPMRExportPtr, |
| PMR_SIZE_T *puiSize, |
| PMR_LOG2ALIGN_T *puiLog2Contig, |
| PMR_PASSWORD_T *puiPassword) |
| { |
| PVR_UNREFERENCED_PARAMETER(psPMR); |
| PVR_UNREFERENCED_PARAMETER(ppsPMRExportPtr); |
| PVR_UNREFERENCED_PARAMETER(puiSize); |
| PVR_UNREFERENCED_PARAMETER(puiLog2Contig); |
| PVR_UNREFERENCED_PARAMETER(puiPassword); |
| |
| return PVRSRV_OK; |
| } |
| |
| |
| PVRSRV_ERROR |
| PMRUnexportPMR(PMR_EXPORT *psPMRExport) |
| { |
| PVR_UNREFERENCED_PARAMETER(psPMRExport); |
| return PVRSRV_OK; |
| } |
| |
| |
| PVRSRV_ERROR |
| PMRImportPMR(PMR_EXPORT *psPMRExport, |
| PMR_PASSWORD_T uiPassword, |
| PMR_SIZE_T uiSize, |
| PMR_LOG2ALIGN_T uiLog2Contig, |
| PMR **ppsPMR) |
| { |
| PVR_UNREFERENCED_PARAMETER(psPMRExport); |
| PVR_UNREFERENCED_PARAMETER(uiPassword); |
| PVR_UNREFERENCED_PARAMETER(uiSize); |
| PVR_UNREFERENCED_PARAMETER(uiLog2Contig); |
| PVR_UNREFERENCED_PARAMETER(ppsPMR); |
| |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| PMRUnimportPMR(PMR *psPMR) |
| { |
| PVR_UNREFERENCED_PARAMETER(psPMR); |
| return PVRSRV_OK; |
| } |
| #endif /* if defined(SUPPORT_INSECURE_EXPORT) */ |
| |
| #if defined(SUPPORT_SECURE_EXPORT) |
| PVRSRV_ERROR PMRSecureUnexportPMR(PMR *psPMR) |
| { |
| _UnrefAndMaybeDestroy(psPMR); |
| return PVRSRV_OK; |
| } |
| |
| static PVRSRV_ERROR _ReleaseSecurePMR(void *psExport) |
| { |
| return PMRSecureUnexportPMR(psExport); |
| } |
| |
| PVRSRV_ERROR PMRSecureExportPMR(CONNECTION_DATA *psConnection, |
| PVRSRV_DEVICE_NODE * psDevNode, |
| PMR *psPMR, |
| IMG_SECURE_TYPE *phSecure, |
| PMR **ppsPMR, |
| CONNECTION_DATA **ppsSecureConnection) |
| { |
| PVRSRV_ERROR eError; |
| |
| PVR_UNREFERENCED_PARAMETER(psDevNode); |
| PVR_UNREFERENCED_PARAMETER(ppsSecureConnection); |
| |
| /* We are acquiring reference to PMR here because OSSecureExport |
| * releases bridge lock and PMR lock for a moment and we don't want PMR |
| * to be removed by other thread in the meantime. */ |
| _Ref(psPMR); |
| |
| eError = OSSecureExport("secure_pmr", |
| _ReleaseSecurePMR, |
| (void *) psPMR, |
| phSecure); |
| |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| *ppsPMR = psPMR; |
| |
| return PVRSRV_OK; |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| _UnrefAndMaybeDestroy(psPMR); |
| return eError; |
| } |
| |
| PVRSRV_ERROR PMRSecureImportPMR(CONNECTION_DATA *psConnection, |
| PVRSRV_DEVICE_NODE *psDevNode, |
| IMG_SECURE_TYPE hSecure, |
| PMR **ppsPMR, |
| IMG_DEVMEM_SIZE_T *puiSize, |
| IMG_DEVMEM_ALIGN_T *puiAlign) |
| { |
| PVRSRV_ERROR eError; |
| PMR *psPMR; |
| |
| PVR_UNREFERENCED_PARAMETER(psConnection); |
| |
| eError = OSSecureImport(hSecure, (void **) &psPMR); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| if (psPMR->psDevNode != psDevNode) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: PMR invalid for this device", __func__)); |
| return PVRSRV_ERROR_PMR_NOT_PERMITTED; |
| } |
| |
| _Ref(psPMR); |
| |
| /* Return the PMR */ |
| *ppsPMR = psPMR; |
| *puiSize = psPMR->uiLogicalSize; |
| *puiAlign = 1 << psPMR->uiLog2ContiguityGuarantee; |
| return PVRSRV_OK; |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR PMRSecureUnimportPMR(PMR *psPMR) |
| { |
| _UnrefAndMaybeDestroy(psPMR); |
| return PVRSRV_OK; |
| } |
| #endif |
| |
| #if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) |
| PVRSRV_ERROR |
| PMRStoreRIHandle(PMR *psPMR, |
| void *hRIHandle) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| |
| psPMR->hRIHandle = hRIHandle; |
| return PVRSRV_OK; |
| } |
| #endif |
| |
| static PVRSRV_ERROR |
| _PMRAcquireKernelMappingData(PMR *psPMR, |
| size_t uiLogicalOffset, |
| size_t uiSize, |
| void **ppvKernelAddressOut, |
| size_t *puiLengthOut, |
| IMG_HANDLE *phPrivOut, |
| IMG_BOOL bMapSparse) |
| { |
| PVRSRV_ERROR eError; |
| void *pvKernelAddress; |
| IMG_HANDLE hPriv; |
| |
| PVR_ASSERT(psPMR != NULL); |
| |
| if (_PMRIsSparse(psPMR) && !bMapSparse) |
| { |
| /* Mapping of sparse allocations must be signalled. */ |
| return PVRSRV_ERROR_PMR_NOT_PERMITTED; |
| } |
| |
| /* Acquire/Release functions must be overridden in pairs */ |
| if (psPMR->psFuncTab->pfnAcquireKernelMappingData == NULL) |
| { |
| PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData == NULL); |
| |
| /* If PMR implementation does not supply this pair of |
| functions, it means they do not permit the PMR to be mapped |
| into kernel memory at all */ |
| eError = PVRSRV_ERROR_PMR_NOT_PERMITTED; |
| goto e0; |
| } |
| PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData != NULL); |
| |
| eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData, |
| uiLogicalOffset, |
| uiSize, |
| &pvKernelAddress, |
| &hPriv, |
| psPMR->uiFlags); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| *ppvKernelAddressOut = pvKernelAddress; |
| if (uiSize == 0) |
| { |
| /* Zero size means map the whole PMR in ...*/ |
| *puiLengthOut = (size_t)psPMR->uiLogicalSize; |
| } |
| else if (uiSize > (1 << psPMR->uiLog2ContiguityGuarantee)) |
| { |
| /* ... map in the requested pages ...*/ |
| *puiLengthOut = uiSize; |
| } |
| else |
| { |
| /* ... otherwise we just map in one page */ |
| *puiLengthOut = 1 << psPMR->uiLog2ContiguityGuarantee; |
| } |
| *phPrivOut = hPriv; |
| |
| return PVRSRV_OK; |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRAcquireKernelMappingData(PMR *psPMR, |
| size_t uiLogicalOffset, |
| size_t uiSize, |
| void **ppvKernelAddressOut, |
| size_t *puiLengthOut, |
| IMG_HANDLE *phPrivOut) |
| { |
| return _PMRAcquireKernelMappingData(psPMR, |
| uiLogicalOffset, |
| uiSize, |
| ppvKernelAddressOut, |
| puiLengthOut, |
| phPrivOut, |
| IMG_FALSE); |
| } |
| |
| PVRSRV_ERROR |
| PMRAcquireSparseKernelMappingData(PMR *psPMR, |
| size_t uiLogicalOffset, |
| size_t uiSize, |
| void **ppvKernelAddressOut, |
| size_t *puiLengthOut, |
| IMG_HANDLE *phPrivOut) |
| { |
| return _PMRAcquireKernelMappingData(psPMR, |
| uiLogicalOffset, |
| uiSize, |
| ppvKernelAddressOut, |
| puiLengthOut, |
| phPrivOut, |
| IMG_TRUE); |
| } |
| |
| PVRSRV_ERROR |
| PMRReleaseKernelMappingData(PMR *psPMR, |
| IMG_HANDLE hPriv) |
| { |
| PVR_ASSERT (psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL); |
| PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData != NULL); |
| |
| psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData, |
| hPriv); |
| |
| return PVRSRV_OK; |
| } |
| |
| #if defined(INTEGRITY_OS) |
| |
| PVRSRV_ERROR |
| PMRMapMemoryObject(PMR *psPMR, |
| IMG_HANDLE *phMemObj, |
| void **pvClientAddr, |
| IMG_HANDLE *phPrivOut) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| IMG_HANDLE hPriv = *phPrivOut; |
| |
| PVR_ASSERT (psPMR->psFuncTab->pfnMapMemoryObject != NULL); |
| |
| eError = psPMR->psFuncTab->pfnMapMemoryObject(hPriv, phMemObj, pvClientAddr, phPrivOut); |
| |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRUnmapMemoryObject(PMR *psPMR, |
| IMG_HANDLE hPriv) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| |
| PVR_ASSERT (psPMR->psFuncTab->pfnUnmapMemoryObject != NULL); |
| |
| eError = psPMR->psFuncTab->pfnUnmapMemoryObject(hPriv); |
| |
| return eError; |
| } |
| |
| #if defined(USING_HYPERVISOR) |
| IMG_HANDLE PMRGetPmr(PMR *psPMR, size_t ulOffset) |
| { |
| PVR_ASSERT(psPMR->psFuncTab->pfnGetPmr != NULL); |
| return psPMR->psFuncTab->pfnGetPmr(psPMR->pvFlavourData, ulOffset); |
| } |
| #endif |
| #endif /* INTEGRITY_OS */ |
| |
| /* |
| _PMRLogicalOffsetToPhysicalOffset |
| |
| Translate between the "logical" offset which the upper levels |
| provide and the physical offset which is what the PMR |
| factories works on. |
| |
| As well as returning the physical offset we return the number of |
| bytes remaining till the next chunk and if this chunk is valid. |
| |
| For multi-page operations, upper layers communicate their |
| Log2PageSize else argument is redundant (set to zero). |
| */ |
| |
| static void |
| _PMRLogicalOffsetToPhysicalOffset(const PMR *psPMR, |
| IMG_UINT32 ui32Log2PageSize, |
| IMG_UINT32 ui32NumOfPages, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_DEVMEM_OFFSET_T *puiPhysicalOffset, |
| IMG_UINT32 *pui32BytesRemain, |
| IMG_BOOL *bValid) |
| { |
| PMR_MAPPING_TABLE *psMappingTable = psPMR->psMappingTable; |
| IMG_DEVMEM_OFFSET_T uiPageSize = 1ULL << ui32Log2PageSize; |
| IMG_DEVMEM_OFFSET_T uiOffset = uiLogicalOffset; |
| IMG_UINT64 ui64ChunkIndex; |
| IMG_UINT32 ui32Remain; |
| IMG_UINT32 idx; |
| |
| /* Must be translating at least a page */ |
| PVR_ASSERT(ui32NumOfPages); |
| |
| if (psMappingTable->ui32NumPhysChunks == psMappingTable->ui32NumVirtChunks) |
| { |
| /* Fast path the common case, as logical and physical offsets are |
| equal we assume the ui32NumOfPages span is also valid */ |
| *pui32BytesRemain = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiOffset); |
| puiPhysicalOffset[0] = uiOffset; |
| bValid[0] = IMG_TRUE; |
| |
| if (ui32NumOfPages > 1) |
| { |
| /* initial offset may not be page aligned, round down */ |
| uiOffset &= ~(uiPageSize-1); |
| for (idx=1; idx < ui32NumOfPages; idx++) |
| { |
| uiOffset += uiPageSize; |
| puiPhysicalOffset[idx] = uiOffset; |
| bValid[idx] = IMG_TRUE; |
| } |
| } |
| } |
| else |
| { |
| for (idx=0; idx < ui32NumOfPages; idx++) |
| { |
| ui64ChunkIndex = OSDivide64r64( |
| uiOffset, |
| TRUNCATE_64BITS_TO_32BITS(psMappingTable->uiChunkSize), |
| &ui32Remain); |
| |
| if (psMappingTable->aui32Translation[ui64ChunkIndex] == TRANSLATION_INVALID) |
| { |
| bValid[idx] = IMG_FALSE; |
| } |
| else |
| { |
| bValid[idx] = IMG_TRUE; |
| } |
| |
| if (idx == 0) |
| { |
| if (ui32Remain == 0) |
| { |
| /* Start of chunk so return the chunk size */ |
| *pui32BytesRemain = TRUNCATE_64BITS_TO_32BITS(psMappingTable->uiChunkSize); |
| } |
| else |
| { |
| *pui32BytesRemain = TRUNCATE_64BITS_TO_32BITS(psMappingTable->uiChunkSize - ui32Remain); |
| } |
| |
| puiPhysicalOffset[idx] = (psMappingTable->aui32Translation[ui64ChunkIndex] * psMappingTable->uiChunkSize) + ui32Remain; |
| |
| /* initial offset may not be page aligned, round down */ |
| uiOffset &= ~(uiPageSize-1); |
| } |
| else |
| { |
| puiPhysicalOffset[idx] = psMappingTable->aui32Translation[ui64ChunkIndex] * psMappingTable->uiChunkSize + ui32Remain; |
| } |
| uiOffset += uiPageSize; |
| } |
| } |
| } |
| |
| static PVRSRV_ERROR |
| _PMR_ReadBytesPhysical(PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiPhysicalOffset, |
| IMG_UINT8 *pcBuffer, |
| size_t uiBufSz, |
| size_t *puiNumBytes) |
| { |
| PVRSRV_ERROR eError; |
| |
| if (psPMR->psFuncTab->pfnReadBytes != NULL) |
| { |
| /* defer to callback if present */ |
| |
| eError = PMRLockSysPhysAddresses(psPMR); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| eError = psPMR->psFuncTab->pfnReadBytes(psPMR->pvFlavourData, |
| uiPhysicalOffset, |
| pcBuffer, |
| uiBufSz, |
| puiNumBytes); |
| PMRUnlockSysPhysAddresses(psPMR); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| } |
| else if (psPMR->psFuncTab->pfnAcquireKernelMappingData) |
| { |
| /* "default" handler for reading bytes */ |
| |
| IMG_HANDLE hKernelMappingHandle; |
| IMG_UINT8 *pcKernelAddress; |
| |
| eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData, |
| (size_t) uiPhysicalOffset, |
| uiBufSz, |
| (void **)&pcKernelAddress, |
| &hKernelMappingHandle, |
| psPMR->uiFlags); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| /* Use the conservative 'DeviceMemCopy' here because we can't know |
| * if this PMR will be mapped cached. |
| */ |
| |
| OSDeviceMemCopy(&pcBuffer[0], pcKernelAddress, uiBufSz); |
| *puiNumBytes = uiBufSz; |
| |
| psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData, |
| hKernelMappingHandle); |
| } |
| else |
| { |
| PVR_DPF((PVR_DBG_ERROR, "PMR_ReadBytes: can't read from this PMR")); |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| OSPanic(); |
| goto e0; |
| } |
| |
| return PVRSRV_OK; |
| |
| /* |
| error exit paths follow |
| */ |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| *puiNumBytes = 0; |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMR_ReadBytes(PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_UINT8 *pcBuffer, |
| size_t uiBufSz, |
| size_t *puiNumBytes) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| IMG_DEVMEM_OFFSET_T uiPhysicalOffset; |
| size_t uiBytesCopied = 0; |
| |
| if (uiLogicalOffset + uiBufSz > psPMR->uiLogicalSize) |
| { |
| uiBufSz = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiLogicalOffset); |
| } |
| PVR_ASSERT(uiBufSz > 0); |
| PVR_ASSERT(uiBufSz <= psPMR->uiLogicalSize); |
| |
| /* |
| PMR implementations can override this. If they don't, a |
| "default" handler uses kernel virtual mappings. If the kernel |
| can't provide a kernel virtual mapping, this function fails |
| */ |
| PVR_ASSERT(psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL || |
| psPMR->psFuncTab->pfnReadBytes != NULL); |
| |
| while (uiBytesCopied != uiBufSz) |
| { |
| IMG_UINT32 ui32Remain; |
| size_t uiBytesToCopy; |
| size_t uiRead; |
| IMG_BOOL bValid; |
| |
| _PMRLogicalOffsetToPhysicalOffset(psPMR, |
| 0, |
| 1, |
| uiLogicalOffset, |
| &uiPhysicalOffset, |
| &ui32Remain, |
| &bValid); |
| /* |
| Copy till either then end of the |
| chunk or end of the buffer |
| */ |
| uiBytesToCopy = MIN(uiBufSz - uiBytesCopied, ui32Remain); |
| |
| if (bValid) |
| { |
| /* Read the data from the PMR */ |
| eError = _PMR_ReadBytesPhysical(psPMR, |
| uiPhysicalOffset, |
| &pcBuffer[uiBytesCopied], |
| uiBytesToCopy, |
| &uiRead); |
| if ((eError != PVRSRV_OK) || (uiRead != uiBytesToCopy)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Failed to read chunk (eError = %s, uiRead = " IMG_SIZE_FMTSPEC " uiBytesToCopy = " IMG_SIZE_FMTSPEC ")", |
| __func__, |
| PVRSRVGetErrorString(eError), |
| uiRead, |
| uiBytesToCopy)); |
| /* Bail out as soon as we hit an error */ |
| break; |
| } |
| } |
| else |
| { |
| PVR_DPF((PVR_DBG_WARNING, |
| "%s: Invalid phys offset at logical offset (" IMG_DEVMEM_OFFSET_FMTSPEC ") logical size (" IMG_DEVMEM_OFFSET_FMTSPEC ")", |
| __func__, |
| uiLogicalOffset, |
| psPMR->uiLogicalSize)); |
| /* Fill invalid chunks with 0 */ |
| OSCachedMemSet(&pcBuffer[uiBytesCopied], 0, uiBytesToCopy); |
| uiRead = uiBytesToCopy; |
| eError = PVRSRV_ERROR_FAILED_TO_GET_PHYS_ADDR; |
| } |
| uiLogicalOffset += uiRead; |
| uiBytesCopied += uiRead; |
| } |
| |
| *puiNumBytes = uiBytesCopied; |
| return eError; |
| } |
| |
| static PVRSRV_ERROR |
| _PMR_WriteBytesPhysical(PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiPhysicalOffset, |
| IMG_UINT8 *pcBuffer, |
| size_t uiBufSz, |
| size_t *puiNumBytes) |
| { |
| PVRSRV_ERROR eError; |
| |
| if (psPMR->psFuncTab->pfnWriteBytes != NULL) |
| { |
| /* defer to callback if present */ |
| |
| eError = PMRLockSysPhysAddresses(psPMR); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| eError = psPMR->psFuncTab->pfnWriteBytes(psPMR->pvFlavourData, |
| uiPhysicalOffset, |
| pcBuffer, |
| uiBufSz, |
| puiNumBytes); |
| PMRUnlockSysPhysAddresses(psPMR); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| } |
| else if (psPMR->psFuncTab->pfnAcquireKernelMappingData) |
| { |
| /* "default" handler for reading bytes */ |
| |
| IMG_HANDLE hKernelMappingHandle; |
| IMG_UINT8 *pcKernelAddress; |
| |
| eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData, |
| (size_t) uiPhysicalOffset, |
| uiBufSz, |
| (void **)&pcKernelAddress, |
| &hKernelMappingHandle, |
| psPMR->uiFlags); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| /* Use the conservative 'DeviceMemCopy' here because we can't know |
| * if this PMR will be mapped cached. |
| */ |
| |
| OSDeviceMemCopy(pcKernelAddress, &pcBuffer[0], uiBufSz); |
| *puiNumBytes = uiBufSz; |
| |
| psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData, |
| hKernelMappingHandle); |
| } |
| else |
| { |
| /* |
| The write callback is optional as it's only required by the debug |
| tools |
| */ |
| PVR_DPF((PVR_DBG_ERROR, "_PMR_WriteBytesPhysical: can't write to this PMR")); |
| eError = PVRSRV_ERROR_PMR_NOT_PERMITTED; |
| OSPanic(); |
| goto e0; |
| } |
| |
| return PVRSRV_OK; |
| |
| /* |
| error exit paths follow |
| */ |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| *puiNumBytes = 0; |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMR_WriteBytes(PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_UINT8 *pcBuffer, |
| size_t uiBufSz, |
| size_t *puiNumBytes) |
| { |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| IMG_DEVMEM_OFFSET_T uiPhysicalOffset; |
| size_t uiBytesCopied = 0; |
| |
| if (uiLogicalOffset + uiBufSz > psPMR->uiLogicalSize) |
| { |
| uiBufSz = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiLogicalOffset); |
| } |
| PVR_ASSERT(uiBufSz > 0); |
| PVR_ASSERT(uiBufSz <= psPMR->uiLogicalSize); |
| |
| /* |
| PMR implementations can override this. If they don't, a |
| "default" handler uses kernel virtual mappings. If the kernel |
| can't provide a kernel virtual mapping, this function fails |
| */ |
| PVR_ASSERT(psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL || |
| psPMR->psFuncTab->pfnWriteBytes != NULL); |
| |
| while (uiBytesCopied != uiBufSz) |
| { |
| IMG_UINT32 ui32Remain; |
| size_t uiBytesToCopy; |
| size_t uiWrite; |
| IMG_BOOL bValid; |
| |
| _PMRLogicalOffsetToPhysicalOffset(psPMR, |
| 0, |
| 1, |
| uiLogicalOffset, |
| &uiPhysicalOffset, |
| &ui32Remain, |
| &bValid); |
| |
| /* |
| Copy till either then end of the |
| chunk or end of the buffer |
| */ |
| uiBytesToCopy = MIN(uiBufSz - uiBytesCopied, ui32Remain); |
| |
| if (bValid) |
| { |
| /* Write the data to the PMR */ |
| eError = _PMR_WriteBytesPhysical(psPMR, |
| uiPhysicalOffset, |
| &pcBuffer[uiBytesCopied], |
| uiBytesToCopy, |
| &uiWrite); |
| if ((eError != PVRSRV_OK) || (uiWrite != uiBytesToCopy)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: Failed to read chunk (eError = %s, uiWrite = " IMG_SIZE_FMTSPEC " uiBytesToCopy = " IMG_SIZE_FMTSPEC ")", |
| __func__, |
| PVRSRVGetErrorString(eError), |
| uiWrite, |
| uiBytesToCopy)); |
| /* Bail out as soon as we hit an error */ |
| break; |
| } |
| } |
| else |
| { |
| /* Ignore writes to invalid pages */ |
| uiWrite = uiBytesToCopy; |
| } |
| uiLogicalOffset += uiWrite; |
| uiBytesCopied += uiWrite; |
| } |
| |
| *puiNumBytes = uiBytesCopied; |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRMMapPMR(PMR *psPMR, PMR_MMAP_DATA pOSMMapData) |
| { |
| if (psPMR->psFuncTab->pfnMMap) |
| { |
| return psPMR->psFuncTab->pfnMMap(psPMR->pvFlavourData, psPMR, pOSMMapData); |
| } |
| |
| return OSMMapPMRGeneric(psPMR, pOSMMapData); |
| } |
| |
| void |
| PMRRefPMR(PMR *psPMR) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| _Ref(psPMR); |
| } |
| |
| PVRSRV_ERROR |
| PMRUnrefPMR(PMR *psPMR) |
| { |
| _UnrefAndMaybeDestroy(psPMR); |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| PMRUnrefUnlockPMR(PMR *psPMR) |
| { |
| PMRUnlockSysPhysAddresses(psPMR); |
| |
| PMRUnrefPMR(psPMR); |
| |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_DEVICE_NODE * |
| PMR_DeviceNode(const PMR *psPMR) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| |
| return psPMR->psDevNode; |
| } |
| |
| PMR_FLAGS_T |
| PMR_Flags(const PMR *psPMR) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| |
| return psPMR->uiFlags; |
| } |
| |
| IMG_BOOL |
| PMR_IsSparse(const PMR *psPMR) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| |
| return _PMRIsSparse(psPMR); |
| } |
| |
| IMG_BOOL |
| PMR_IsUnpinned(const PMR *psPMR) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| |
| return psPMR->bIsUnpinned; |
| } |
| |
| PVRSRV_ERROR |
| PMR_LogicalSize(const PMR *psPMR, |
| IMG_DEVMEM_SIZE_T *puiLogicalSize) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| |
| *puiLogicalSize = psPMR->uiLogicalSize; |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| PMR_PhysicalSize(const PMR *psPMR, |
| IMG_DEVMEM_SIZE_T *puiPhysicalSize) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| |
| /* iLockCount will be > 0 for any backed PMR (backed on demand or not) */ |
| if ((OSAtomicRead(&psPMR->iLockCount) > 0) && !psPMR->bIsUnpinned) |
| { |
| if (psPMR->bSparseAlloc) |
| { |
| *puiPhysicalSize = psPMR->psMappingTable->uiChunkSize * psPMR->psMappingTable->ui32NumPhysChunks; |
| } |
| else |
| { |
| *puiPhysicalSize = psPMR->uiLogicalSize; |
| } |
| } |
| else |
| { |
| *puiPhysicalSize = 0; |
| } |
| return PVRSRV_OK; |
| } |
| |
| PHYS_HEAP * |
| PMR_PhysHeap(const PMR *psPMR) |
| { |
| return psPMR->psPhysHeap; |
| } |
| |
| PVRSRV_ERROR |
| PMR_IsOffsetValid(const PMR *psPMR, |
| IMG_UINT32 ui32Log2PageSize, |
| IMG_UINT32 ui32NumOfPages, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_BOOL *pbValid) |
| { |
| IMG_DEVMEM_OFFSET_T auiPhysicalOffset[PMR_MAX_TRANSLATION_STACK_ALLOC]; |
| IMG_UINT32 aui32BytesRemain[PMR_MAX_TRANSLATION_STACK_ALLOC]; |
| IMG_DEVMEM_OFFSET_T *puiPhysicalOffset = auiPhysicalOffset; |
| IMG_UINT32 *pui32BytesRemain = aui32BytesRemain; |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| |
| PVR_ASSERT(psPMR != NULL); |
| PVR_ASSERT(psPMR->uiLogicalSize >= uiLogicalOffset); |
| |
| if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC) |
| { |
| puiPhysicalOffset = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEVMEM_OFFSET_T)); |
| if (puiPhysicalOffset == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e0; |
| } |
| |
| pui32BytesRemain = OSAllocMem(ui32NumOfPages * sizeof(IMG_UINT32)); |
| if (pui32BytesRemain == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e0; |
| } |
| } |
| |
| _PMRLogicalOffsetToPhysicalOffset(psPMR, |
| ui32Log2PageSize, |
| ui32NumOfPages, |
| uiLogicalOffset, |
| puiPhysicalOffset, |
| pui32BytesRemain, |
| pbValid); |
| |
| e0: |
| if (puiPhysicalOffset != auiPhysicalOffset && puiPhysicalOffset != NULL) |
| { |
| OSFreeMem(puiPhysicalOffset); |
| } |
| |
| if (pui32BytesRemain != aui32BytesRemain && pui32BytesRemain != NULL) |
| { |
| OSFreeMem(pui32BytesRemain); |
| } |
| |
| return eError; |
| } |
| |
| PMR_MAPPING_TABLE * |
| PMR_GetMappigTable(const PMR *psPMR) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| return psPMR->psMappingTable; |
| |
| } |
| |
| IMG_UINT32 |
| PMR_GetLog2Contiguity(const PMR *psPMR) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| return psPMR->uiLog2ContiguityGuarantee; |
| } |
| |
| const IMG_CHAR * |
| PMR_GetAnnotation(const PMR *psPMR) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| return psPMR->szAnnotation; |
| } |
| |
| PMR_IMPL_TYPE |
| PMR_GetType(const PMR *psPMR) |
| { |
| PVR_ASSERT(psPMR != NULL); |
| return psPMR->eFlavour; |
| } |
| |
| /* must have called PMRLockSysPhysAddresses() before calling this! */ |
| PVRSRV_ERROR |
| PMR_DevPhysAddr(const PMR *psPMR, |
| IMG_UINT32 ui32Log2PageSize, |
| IMG_UINT32 ui32NumOfPages, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_DEV_PHYADDR *psDevAddrPtr, |
| IMG_BOOL *pbValid) |
| { |
| IMG_UINT32 ui32Remain; |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| IMG_DEVMEM_OFFSET_T auiPhysicalOffset[PMR_MAX_TRANSLATION_STACK_ALLOC]; |
| IMG_DEVMEM_OFFSET_T *puiPhysicalOffset = auiPhysicalOffset; |
| |
| PVR_ASSERT(psPMR != NULL); |
| PVR_ASSERT(ui32NumOfPages > 0); |
| PVR_ASSERT(psPMR->psFuncTab->pfnDevPhysAddr != NULL); |
| |
| #ifdef PVRSRV_NEED_PVR_ASSERT |
| PVR_ASSERT(OSAtomicRead(&psPMR->iLockCount) > (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1)); |
| #endif |
| |
| if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC) |
| { |
| puiPhysicalOffset = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEVMEM_OFFSET_T)); |
| if (puiPhysicalOffset == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e0; |
| } |
| } |
| |
| _PMRLogicalOffsetToPhysicalOffset(psPMR, |
| ui32Log2PageSize, |
| ui32NumOfPages, |
| uiLogicalOffset, |
| puiPhysicalOffset, |
| &ui32Remain, |
| pbValid); |
| if (*pbValid || _PMRIsSparse(psPMR)) |
| { |
| /* Sparse PMR may not always have the first page valid */ |
| eError = psPMR->psFuncTab->pfnDevPhysAddr(psPMR->pvFlavourData, |
| ui32Log2PageSize, |
| ui32NumOfPages, |
| puiPhysicalOffset, |
| pbValid, |
| psDevAddrPtr); |
| #if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES) |
| /* Currently excluded from the default build because of performance concerns. |
| * We do not need this part in all systems because the GPU has the same address view of system RAM as the CPU. |
| * Alternatively this could be implemented as part of the PMR-factories directly */ |
| |
| if (PhysHeapGetType(psPMR->psPhysHeap) == PHYS_HEAP_TYPE_UMA || |
| PhysHeapGetType(psPMR->psPhysHeap) == PHYS_HEAP_TYPE_DMA) |
| { |
| IMG_UINT32 i; |
| IMG_DEV_PHYADDR sDevPAddrCorrected; |
| |
| /* Copy the translated addresses to the correct array */ |
| for (i = 0; i < ui32NumOfPages; i++) |
| { |
| PhysHeapCpuPAddrToDevPAddr(psPMR->psPhysHeap, |
| 1, |
| &sDevPAddrCorrected, |
| (IMG_CPU_PHYADDR *) &psDevAddrPtr[i]); |
| psDevAddrPtr[i].uiAddr = sDevPAddrCorrected.uiAddr; |
| } |
| } |
| #endif |
| } |
| |
| if (puiPhysicalOffset != auiPhysicalOffset) |
| { |
| OSFreeMem(puiPhysicalOffset); |
| } |
| |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| return PVRSRV_OK; |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMR_CpuPhysAddr(const PMR *psPMR, |
| IMG_UINT32 ui32Log2PageSize, |
| IMG_UINT32 ui32NumOfPages, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_CPU_PHYADDR *psCpuAddrPtr, |
| IMG_BOOL *pbValid) |
| { |
| PVRSRV_ERROR eError; |
| IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC]; |
| IMG_DEV_PHYADDR *psDevPAddr = asDevPAddr; |
| |
| if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC) |
| { |
| psDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR)); |
| if (psDevPAddr == NULL) |
| { |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto e0; |
| } |
| } |
| |
| eError = PMR_DevPhysAddr(psPMR, ui32Log2PageSize, ui32NumOfPages, |
| uiLogicalOffset, psDevPAddr, pbValid); |
| if (eError != PVRSRV_OK) |
| { |
| goto e1; |
| } |
| PhysHeapDevPAddrToCpuPAddr(psPMR->psPhysHeap, ui32NumOfPages, psCpuAddrPtr, psDevPAddr); |
| |
| if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC) |
| { |
| OSFreeMem(psDevPAddr); |
| } |
| |
| return PVRSRV_OK; |
| e1: |
| if (psDevPAddr != asDevPAddr) |
| { |
| OSFreeMem(psDevPAddr); |
| } |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR PMR_ChangeSparseMem(PMR *psPMR, |
| IMG_UINT32 ui32AllocPageCount, |
| IMG_UINT32 *pai32AllocIndices, |
| IMG_UINT32 ui32FreePageCount, |
| IMG_UINT32 *pai32FreeIndices, |
| IMG_UINT32 uiSparseFlags) |
| { |
| PVRSRV_ERROR eError; |
| |
| if (NULL == psPMR->psFuncTab->pfnChangeSparseMem) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: This type of sparse PMR cannot be changed.", |
| __func__)); |
| return PVRSRV_ERROR_NOT_IMPLEMENTED; |
| } |
| |
| eError = psPMR->psFuncTab->pfnChangeSparseMem(psPMR->pvFlavourData, |
| psPMR, |
| ui32AllocPageCount, |
| pai32AllocIndices, |
| ui32FreePageCount, |
| pai32FreeIndices, |
| uiSparseFlags); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| #if defined(PDUMP) |
| { |
| IMG_BOOL bInitialise = IMG_FALSE; |
| IMG_UINT32 ui32InitValue = 0; |
| |
| if (PVRSRV_CHECK_ZERO_ON_ALLOC(PMR_Flags(psPMR))) |
| { |
| bInitialise = IMG_TRUE; |
| } |
| else if (PVRSRV_CHECK_POISON_ON_ALLOC(PMR_Flags(psPMR))) |
| { |
| ui32InitValue = 0xDEADBEEF; |
| bInitialise = IMG_TRUE; |
| } |
| |
| PDumpPMRChangeSparsePMR(psPMR, |
| 1 << psPMR->uiLog2ContiguityGuarantee, |
| ui32AllocPageCount, |
| pai32AllocIndices, |
| ui32FreePageCount, |
| pai32FreeIndices, |
| bInitialise, |
| ui32InitValue, |
| &psPMR->hPDumpAllocHandle); |
| } |
| |
| #endif |
| |
| e0: |
| return eError; |
| } |
| |
| |
| PVRSRV_ERROR PMR_ChangeSparseMemCPUMap(PMR *psPMR, |
| IMG_UINT64 sCpuVAddrBase, |
| IMG_UINT32 ui32AllocPageCount, |
| IMG_UINT32 *pai32AllocIndices, |
| IMG_UINT32 ui32FreePageCount, |
| IMG_UINT32 *pai32FreeIndices) |
| { |
| PVRSRV_ERROR eError; |
| |
| if ((NULL == psPMR->psFuncTab) || |
| (NULL == psPMR->psFuncTab->pfnChangeSparseMemCPUMap)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "%s: This type of sparse PMR cannot be changed.", |
| __func__)); |
| return PVRSRV_ERROR_NOT_IMPLEMENTED; |
| } |
| |
| eError = psPMR->psFuncTab->pfnChangeSparseMemCPUMap(psPMR->pvFlavourData, |
| psPMR, |
| sCpuVAddrBase, |
| ui32AllocPageCount, |
| pai32AllocIndices, |
| ui32FreePageCount, |
| pai32FreeIndices); |
| |
| return eError; |
| } |
| |
| |
| |
| #if defined(PDUMP) |
| |
| static PVRSRV_ERROR |
| _PMR_PDumpSymbolicAddrPhysical(const PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiPhysicalOffset, |
| IMG_UINT32 ui32MemspaceNameLen, |
| IMG_CHAR *pszMemspaceName, |
| IMG_UINT32 ui32SymbolicAddrLen, |
| IMG_CHAR *pszSymbolicAddr, |
| IMG_DEVMEM_OFFSET_T *puiNewOffset, |
| IMG_DEVMEM_OFFSET_T *puiNextSymName) |
| { |
| if (DevmemCPUCacheCoherency(psPMR->psDevNode, psPMR->uiFlags) || |
| DevmemDeviceCacheCoherency(psPMR->psDevNode, psPMR->uiFlags)) |
| { |
| OSSNPrintf(pszMemspaceName, |
| ui32MemspaceNameLen, |
| PMR_MEMSPACE_CACHE_COHERENT_FMTSPEC, |
| psPMR->pszPDumpDefaultMemspaceName); |
| } |
| else |
| { |
| OSSNPrintf(pszMemspaceName, ui32MemspaceNameLen, PMR_MEMSPACE_FMTSPEC, |
| psPMR->pszPDumpDefaultMemspaceName); |
| } |
| |
| OSSNPrintf(pszSymbolicAddr, |
| ui32SymbolicAddrLen, |
| PMR_SYMBOLICADDR_FMTSPEC, |
| PMR_DEFAULT_PREFIX, |
| psPMR->uiSerialNum, |
| uiPhysicalOffset >> PMR_GetLog2Contiguity(psPMR), |
| psPMR->szAnnotation); |
| PDumpMakeStringValid(pszSymbolicAddr, OSStringLength(pszSymbolicAddr)); |
| |
| |
| *puiNewOffset = uiPhysicalOffset & ((1 << PMR_GetLog2Contiguity(psPMR))-1); |
| *puiNextSymName = (IMG_DEVMEM_OFFSET_T) (((uiPhysicalOffset >> PMR_GetLog2Contiguity(psPMR))+1) |
| << PMR_GetLog2Contiguity(psPMR)); |
| |
| return PVRSRV_OK; |
| } |
| |
| |
| PVRSRV_ERROR |
| PMR_PDumpSymbolicAddr(const PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_UINT32 ui32MemspaceNameLen, |
| IMG_CHAR *pszMemspaceName, |
| IMG_UINT32 ui32SymbolicAddrLen, |
| IMG_CHAR *pszSymbolicAddr, |
| IMG_DEVMEM_OFFSET_T *puiNewOffset, |
| IMG_DEVMEM_OFFSET_T *puiNextSymName |
| ) |
| { |
| IMG_DEVMEM_OFFSET_T uiPhysicalOffset; |
| IMG_UINT32 ui32Remain; |
| IMG_BOOL bValid; |
| |
| PVR_ASSERT(uiLogicalOffset < psPMR->uiLogicalSize); |
| |
| _PMRLogicalOffsetToPhysicalOffset(psPMR, |
| 0, |
| 1, |
| uiLogicalOffset, |
| &uiPhysicalOffset, |
| &ui32Remain, |
| &bValid); |
| |
| if (!bValid) |
| { |
| /* For sparse allocations, for a given logical address, there may not be a |
| * physical memory backing, the virtual range can still be valid. |
| */ |
| uiPhysicalOffset = uiLogicalOffset; |
| } |
| |
| return _PMR_PDumpSymbolicAddrPhysical(psPMR, |
| uiPhysicalOffset, |
| ui32MemspaceNameLen, |
| pszMemspaceName, |
| ui32SymbolicAddrLen, |
| pszSymbolicAddr, |
| puiNewOffset, |
| puiNextSymName); |
| } |
| |
| /*! |
| * @brief Writes a WRW command to the script2 buffer, representing a |
| * dword write to a physical allocation. Size is always |
| * sizeof(IMG_UINT32). |
| * @param psPMR - PMR object representing allocation |
| * @param uiLogicalOffset - offset |
| * @param ui32Value - value to write |
| * @param uiPDumpFlags - pdump flags |
| * @return PVRSRV_ERROR |
| */ |
| PVRSRV_ERROR |
| PMRPDumpLoadMemValue32(PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_UINT32 ui32Value, |
| PDUMP_FLAGS_T uiPDumpFlags) |
| { |
| PVRSRV_ERROR eError; |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName; |
| IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee; |
| |
| PVR_ASSERT(uiLogicalOffset + sizeof(ui32Value) <= psPMR->uiLogicalSize); |
| /* Especially make sure to not cross a block boundary */ |
| PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value)) |
| <= uiPMRPageSize)); |
| |
| eError = PMRLockSysPhysAddresses(psPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| /* Get the symbolic address of the PMR */ |
| eError = PMR_PDumpSymbolicAddr(psPMR, |
| uiLogicalOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiPDumpSymbolicOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| /* Write the WRW script command */ |
| eError = PDumpPMRWRW32(aszMemspaceName, |
| aszSymbolicName, |
| uiPDumpSymbolicOffset, |
| ui32Value, |
| uiPDumpFlags); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| eError = PMRUnlockSysPhysAddresses(psPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| return PVRSRV_OK; |
| } |
| |
| /*! |
| * @brief Writes a RDW followed by a WRW command to the pdump script to perform |
| * an effective copy from memory to memory. Memory copied is of size |
| * sizeof (IMG_UINT32) |
| * |
| * @param psDstPMR - PMR object representing allocation of destination |
| * @param uiDstLogicalOffset - destination offset |
| * @param psSrcPMR - PMR object representing allocation of source |
| * @param uiSrcLogicalOffset - source offset |
| * @param pszTmpVar - pdump temporary variable used during the copy |
| * @param uiPDumpFlags - pdump flags |
| * @return PVRSRV_ERROR |
| */ |
| PVRSRV_ERROR |
| PMRPDumpCopyMem32(PMR *psDstPMR, |
| IMG_DEVMEM_OFFSET_T uiDstLogicalOffset, |
| PMR *psSrcPMR, |
| IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset, |
| const IMG_CHAR *pszTmpVar, |
| PDUMP_FLAGS_T uiPDumpFlags) |
| { |
| PVRSRV_ERROR eError; |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName; |
| const IMG_UINT32 uiDstPMRPageSize = 1 << psDstPMR->uiLog2ContiguityGuarantee; |
| const IMG_UINT32 uiSrcPMRPageSize = 1 << psSrcPMR->uiLog2ContiguityGuarantee; |
| |
| PVR_ASSERT(uiSrcLogicalOffset + sizeof(IMG_UINT32) <= psSrcPMR->uiLogicalSize); |
| /* Especially make sure to not cross a block boundary */ |
| PVR_ASSERT(( ((uiSrcLogicalOffset & (uiSrcPMRPageSize-1)) + sizeof(IMG_UINT32)) |
| <= uiSrcPMRPageSize)); |
| |
| PVR_ASSERT(uiDstLogicalOffset + sizeof(IMG_UINT32) <= psDstPMR->uiLogicalSize); |
| /* Especially make sure to not cross a block boundary */ |
| PVR_ASSERT(( ((uiDstLogicalOffset & (uiDstPMRPageSize-1)) + sizeof(IMG_UINT32)) |
| <= uiDstPMRPageSize)); |
| |
| |
| eError = PMRLockSysPhysAddresses(psSrcPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| /* Get the symbolic address of the source PMR */ |
| eError = PMR_PDumpSymbolicAddr(psSrcPMR, |
| uiSrcLogicalOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiPDumpSymbolicOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| /* Issue PDump read command */ |
| eError = PDumpPMRRDW32MemToInternalVar(pszTmpVar, |
| aszMemspaceName, |
| aszSymbolicName, |
| uiPDumpSymbolicOffset, |
| uiPDumpFlags); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| eError = PMRUnlockSysPhysAddresses(psSrcPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| |
| |
| eError = PMRLockSysPhysAddresses(psDstPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| |
| /* Get the symbolic address of the destination PMR */ |
| eError = PMR_PDumpSymbolicAddr(psDstPMR, |
| uiDstLogicalOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiPDumpSymbolicOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| |
| /* Write the WRW script command */ |
| eError = PDumpPMRWRW32InternalVarToMem(aszMemspaceName, |
| aszSymbolicName, |
| uiPDumpSymbolicOffset, |
| pszTmpVar, |
| uiPDumpFlags); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| |
| eError = PMRUnlockSysPhysAddresses(psDstPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| return PVRSRV_OK; |
| } |
| |
| /*! |
| * @brief Writes a WRW64 command to the script2 buffer, representing a |
| * dword write to a physical allocation. Size is always |
| * sizeof(IMG_UINT64). |
| * @param psPMR - PMR object representing allocation |
| * @param uiLogicalOffset - offset |
| * @param ui64Value - value to write |
| * @param uiPDumpFlags - pdump flags |
| * @return PVRSRV_ERROR |
| */ |
| PVRSRV_ERROR |
| PMRPDumpLoadMemValue64(PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_UINT64 ui64Value, |
| PDUMP_FLAGS_T uiPDumpFlags) |
| { |
| PVRSRV_ERROR eError; |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName; |
| IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee; |
| |
| |
| PVR_ASSERT(uiLogicalOffset + sizeof(ui64Value) <= psPMR->uiLogicalSize); |
| /* Especially make sure to not cross a block boundary */ |
| PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui64Value)) |
| <= uiPMRPageSize)); |
| |
| eError = PMRLockSysPhysAddresses(psPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| /* Get the symbolic address of the PMR */ |
| eError = PMR_PDumpSymbolicAddr(psPMR, |
| uiLogicalOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiPDumpSymbolicOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| /* Write the WRW script command */ |
| eError = PDumpPMRWRW64(aszMemspaceName, |
| aszSymbolicName, |
| uiPDumpSymbolicOffset, |
| ui64Value, |
| uiPDumpFlags); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| eError = PMRUnlockSysPhysAddresses(psPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| return PVRSRV_OK; |
| } |
| |
| /*! |
| * @brief Writes a RDW64 followed by a WRW64 command to the pdump script to |
| * perform an effective copy from memory to memory. Memory copied is of |
| * size sizeof (IMG_UINT32) |
| * |
| * @param psDstPMR - PMR object representing allocation of destination |
| * @param uiDstLogicalOffset - destination offset |
| * @param psSrcPMR - PMR object representing allocation of source |
| * @param uiSrcLogicalOffset - source offset |
| * @param pszTmpVar - pdump temporary variable used during the copy |
| * @param uiPDumpFlags - pdump flags |
| * @return PVRSRV_ERROR |
| */ |
| PVRSRV_ERROR |
| PMRPDumpCopyMem64(PMR *psDstPMR, |
| IMG_DEVMEM_OFFSET_T uiDstLogicalOffset, |
| PMR *psSrcPMR, |
| IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset, |
| const IMG_CHAR *pszTmpVar, |
| PDUMP_FLAGS_T uiPDumpFlags) |
| { |
| PVRSRV_ERROR eError; |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName; |
| const IMG_UINT32 uiDstPMRPageSize = 1 << psDstPMR->uiLog2ContiguityGuarantee; |
| const IMG_UINT32 uiSrcPMRPageSize = 1 << psSrcPMR->uiLog2ContiguityGuarantee; |
| |
| PVR_ASSERT(uiSrcLogicalOffset + sizeof(IMG_UINT32) <= psSrcPMR->uiLogicalSize); |
| /* Especially make sure to not cross a block boundary */ |
| PVR_ASSERT(( ((uiSrcLogicalOffset & (uiSrcPMRPageSize-1)) + sizeof(IMG_UINT32)) |
| <= uiSrcPMRPageSize)); |
| |
| PVR_ASSERT(uiDstLogicalOffset + sizeof(IMG_UINT32) <= psDstPMR->uiLogicalSize); |
| /* Especially make sure to not cross a block boundary */ |
| PVR_ASSERT(( ((uiDstLogicalOffset & (uiDstPMRPageSize-1)) + sizeof(IMG_UINT32)) |
| <= uiDstPMRPageSize)); |
| |
| |
| eError = PMRLockSysPhysAddresses(psSrcPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| /* Get the symbolic address of the source PMR */ |
| eError = PMR_PDumpSymbolicAddr(psSrcPMR, |
| uiSrcLogicalOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiPDumpSymbolicOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| /* Issue PDump read command */ |
| eError = PDumpPMRRDW64MemToInternalVar(pszTmpVar, |
| aszMemspaceName, |
| aszSymbolicName, |
| uiPDumpSymbolicOffset, |
| uiPDumpFlags); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| eError = PMRUnlockSysPhysAddresses(psSrcPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| |
| |
| eError = PMRLockSysPhysAddresses(psDstPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| |
| /* Get the symbolic address of the destination PMR */ |
| eError = PMR_PDumpSymbolicAddr(psDstPMR, |
| uiDstLogicalOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiPDumpSymbolicOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| |
| /* Write the WRW script command */ |
| eError = PDumpPMRWRW64InternalVarToMem(aszMemspaceName, |
| aszSymbolicName, |
| uiPDumpSymbolicOffset, |
| pszTmpVar, |
| uiPDumpFlags); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| |
| eError = PMRUnlockSysPhysAddresses(psDstPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| return PVRSRV_OK; |
| } |
| |
| /*! |
| * @brief PDumps the contents of the given allocation. |
| * If bZero is IMG_TRUE then the zero page in the parameter stream is used |
| * as the source of data, rather than the allocation's actual backing. |
| * @param psPMR - PMR object representing allocation |
| * @param uiLogicalOffset - Offset to write at |
| * @param uiSize - Number of bytes to write |
| * @param uiPDumpFlags - PDump flags |
| * @param bZero - Use the PDump zero page as the source |
| * @return PVRSRV_ERROR |
| */ |
| PVRSRV_ERROR |
| PMRPDumpLoadMem(PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_DEVMEM_SIZE_T uiSize, |
| PDUMP_FLAGS_T uiPDumpFlags, |
| IMG_BOOL bZero) |
| { |
| PVRSRV_ERROR eError; |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiOutOffset; |
| IMG_DEVMEM_OFFSET_T uiCurrentOffset = uiLogicalOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName = 0; |
| const IMG_CHAR *pszParamStreamFileName; |
| PDUMP_FILEOFFSET_T uiParamStreamFileOffset; |
| |
| /* required when !bZero */ |
| #define PMR_MAX_PDUMP_BUFSZ (1<<21) |
| IMG_CHAR aszParamStreamFilename[PDUMP_PARAM_MAX_FILE_NAME]; |
| IMG_UINT8 *pcBuffer = NULL; |
| size_t uiBufSz; |
| IMG_BOOL bValid; |
| IMG_DEVMEM_SIZE_T uiSizeRemain = uiSize; |
| |
| PVR_ASSERT(uiLogicalOffset + uiSize <= psPMR->uiLogicalSize); |
| |
| /* Get the correct PDump stream file name */ |
| if (bZero) |
| { |
| PDumpCommentWithFlags(uiPDumpFlags, |
| "Zeroing allocation (" IMG_DEVMEM_SIZE_FMTSPEC " bytes)", |
| uiSize); |
| |
| /* get the zero page information. it is constant for this function */ |
| PDumpGetParameterZeroPageInfo(&uiParamStreamFileOffset, |
| &uiBufSz, |
| &pszParamStreamFileName); |
| } |
| else |
| { |
| |
| uiBufSz = 1 << PMR_GetLog2Contiguity(psPMR); |
| PVR_ASSERT((1 << PMR_GetLog2Contiguity(psPMR)) <= PMR_MAX_PDUMP_BUFSZ); |
| |
| pcBuffer = OSAllocMem(uiBufSz); |
| |
| PVR_LOGR_IF_NOMEM(pcBuffer, "OSAllocMem"); |
| |
| eError = PMRLockSysPhysAddresses(psPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| pszParamStreamFileName = aszParamStreamFilename; |
| } |
| |
| /* Loop over all touched symbolic addresses of the PMR and |
| * emit LDBs to load the contents. */ |
| while (uiCurrentOffset < (uiLogicalOffset + uiSize)) |
| { |
| /* Get the correct symbolic name for the current offset */ |
| eError = PMR_PDumpSymbolicAddr(psPMR, |
| uiCurrentOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiOutOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| PVR_ASSERT((uiNextSymName - uiCurrentOffset) <= uiBufSz); |
| |
| PMR_IsOffsetValid(psPMR, |
| 0, |
| 1, |
| uiCurrentOffset, |
| &bValid); |
| |
| /* Either just LDB the zeros or read from the PMR and store that |
| * in the pdump stream */ |
| if (bValid) |
| { |
| size_t uiNumBytes; |
| |
| if (bZero) |
| { |
| uiNumBytes = MIN(uiSizeRemain, uiNextSymName - uiCurrentOffset); |
| } |
| else |
| { |
| IMG_DEVMEM_OFFSET_T uiReadOffset; |
| uiReadOffset = ((uiNextSymName > (uiLogicalOffset + uiSize)) ? |
| uiLogicalOffset + uiSize - uiCurrentOffset : |
| uiNextSymName - uiCurrentOffset); |
| |
| eError = PMR_ReadBytes(psPMR, |
| uiCurrentOffset, |
| pcBuffer, |
| uiReadOffset, |
| &uiNumBytes); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| eError = PDumpWriteParameterBlob(pcBuffer, |
| uiNumBytes, |
| uiPDumpFlags, |
| &aszParamStreamFilename[0], |
| sizeof(aszParamStreamFilename), |
| &uiParamStreamFileOffset); |
| if (eError == PVRSRV_ERROR_PDUMP_NOT_ALLOWED) |
| { |
| /* Write to parameter file prevented under the flags and |
| * current state of the driver so skip further writes. |
| */ |
| eError = PVRSRV_OK; |
| } |
| else if (eError != PVRSRV_OK) |
| { |
| PDUMP_ERROR(eError, "Failed to write PMR memory to parameter file"); |
| } |
| } |
| |
| /* Emit the LDB command to the current symbolic address*/ |
| eError = PDumpPMRLDB(aszMemspaceName, |
| aszSymbolicName, |
| uiOutOffset, |
| uiNumBytes, |
| pszParamStreamFileName, |
| uiParamStreamFileOffset, |
| uiPDumpFlags); |
| uiSizeRemain = uiSizeRemain - uiNumBytes; |
| } |
| uiCurrentOffset = uiNextSymName; |
| } |
| |
| if (!bZero) |
| { |
| eError = PMRUnlockSysPhysAddresses(psPMR); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| OSFreeMem(pcBuffer); |
| } |
| |
| return PVRSRV_OK; |
| } |
| |
| |
| |
| PVRSRV_ERROR |
| PMRPDumpSaveToFile(const PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_DEVMEM_SIZE_T uiSize, |
| IMG_UINT32 uiArraySize, |
| const IMG_CHAR *pszFilename, |
| IMG_UINT32 uiFileOffset) |
| { |
| PVRSRV_ERROR eError; |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiOutOffset; |
| IMG_DEVMEM_OFFSET_T uiCurrentOffset = uiLogicalOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName = 0; |
| IMG_UINT32 uiCurrentFileOffset = uiFileOffset; |
| |
| PVR_UNREFERENCED_PARAMETER(uiArraySize); |
| |
| PVR_ASSERT(uiLogicalOffset + uiSize <= psPMR->uiLogicalSize); |
| |
| while (uiCurrentOffset < (uiLogicalOffset + uiSize)) |
| { |
| IMG_DEVMEM_OFFSET_T uiReadOffset; |
| |
| eError = PMR_PDumpSymbolicAddr(psPMR, |
| uiCurrentOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiOutOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| PVR_ASSERT(uiNextSymName <= psPMR->uiLogicalSize); |
| |
| uiReadOffset = ((uiNextSymName > (uiLogicalOffset + uiSize)) ? |
| uiLogicalOffset + uiSize - uiCurrentOffset : |
| uiNextSymName - uiCurrentOffset); |
| |
| eError = PDumpPMRSAB(aszMemspaceName, |
| aszSymbolicName, |
| uiOutOffset, |
| uiReadOffset, |
| pszFilename, |
| uiCurrentFileOffset); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| uiCurrentFileOffset += uiNextSymName - uiCurrentOffset; |
| uiCurrentOffset = uiNextSymName; |
| } |
| |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| PMRPDumpPol32(const PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiLogicalOffset, |
| IMG_UINT32 ui32Value, |
| IMG_UINT32 ui32Mask, |
| PDUMP_POLL_OPERATOR eOperator, |
| PDUMP_FLAGS_T uiPDumpFlags) |
| { |
| PVRSRV_ERROR eError; |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiPDumpOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName; |
| IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee; |
| |
| /* Make sure to not cross a block boundary */ |
| PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value)) |
| <= uiPMRPageSize)); |
| |
| eError = PMR_PDumpSymbolicAddr(psPMR, |
| uiLogicalOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiPDumpOffset, |
| &uiNextSymName); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| #define _MEMPOLL_DELAY (1000) |
| #define _MEMPOLL_COUNT (2000000000 / _MEMPOLL_DELAY) |
| |
| eError = PDumpPMRPOL(aszMemspaceName, |
| aszSymbolicName, |
| uiPDumpOffset, |
| ui32Value, |
| ui32Mask, |
| eOperator, |
| _MEMPOLL_COUNT, |
| _MEMPOLL_DELAY, |
| uiPDumpFlags); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| return PVRSRV_OK; |
| |
| /* |
| error exit paths follow |
| */ |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRPDumpCBP(const PMR *psPMR, |
| IMG_DEVMEM_OFFSET_T uiReadOffset, |
| IMG_DEVMEM_OFFSET_T uiWriteOffset, |
| IMG_DEVMEM_SIZE_T uiPacketSize, |
| IMG_DEVMEM_SIZE_T uiBufferSize) |
| { |
| PVRSRV_ERROR eError; |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiPDumpOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName; |
| |
| eError = PMR_PDumpSymbolicAddr(psPMR, |
| uiReadOffset, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiPDumpOffset, |
| &uiNextSymName); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| eError = PDumpPMRCBP(aszMemspaceName, |
| aszSymbolicName, |
| uiPDumpOffset, |
| uiWriteOffset, |
| uiPacketSize, |
| uiBufferSize); |
| if (eError != PVRSRV_OK) |
| { |
| goto e0; |
| } |
| |
| return PVRSRV_OK; |
| |
| /* |
| error exit paths follow |
| */ |
| |
| e0: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| void |
| PDumpPMRChangeSparsePMR(PMR *psPMR, |
| IMG_UINT32 uiBlockSize, |
| IMG_UINT32 ui32AllocPageCount, |
| IMG_UINT32 *pai32AllocIndices, |
| IMG_UINT32 ui32FreePageCount, |
| IMG_UINT32 *pai32FreeIndices, |
| IMG_BOOL bInitialise, |
| IMG_UINT32 ui32InitValue, |
| IMG_HANDLE *phPDumpAllocInfoOut) |
| { |
| PVRSRV_ERROR eError; |
| IMG_HANDLE *phPDumpAllocInfo = (IMG_HANDLE*) psPMR->hPDumpAllocHandle; |
| |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName; |
| IMG_UINT32 i, uiIndex; |
| |
| /* Remove pages from the PMR */ |
| for (i = 0; i < ui32FreePageCount; i++) |
| { |
| uiIndex = pai32FreeIndices[i]; |
| |
| eError = PDumpFree(phPDumpAllocInfo[uiIndex]); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| phPDumpAllocInfo[uiIndex] = NULL; |
| } |
| |
| /* Add new pages to the PMR */ |
| for (i = 0; i < ui32AllocPageCount; i++) |
| { |
| uiIndex = pai32AllocIndices[i]; |
| |
| PVR_ASSERT(phPDumpAllocInfo[uiIndex] == NULL); |
| |
| eError = PMR_PDumpSymbolicAddr(psPMR, |
| uiIndex * uiBlockSize, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| eError = PDumpMalloc(aszMemspaceName, |
| aszSymbolicName, |
| uiBlockSize, |
| uiBlockSize, |
| bInitialise, |
| ui32InitValue, |
| &phPDumpAllocInfo[uiIndex], |
| PDUMP_NONE); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| } |
| |
| /* (IMG_HANDLE) <- (IMG_HANDLE*) */ |
| *phPDumpAllocInfoOut = (IMG_HANDLE) phPDumpAllocInfo; |
| } |
| |
| void |
| PDumpPMRFreePMR(PMR *psPMR, |
| IMG_DEVMEM_SIZE_T uiSize, |
| IMG_DEVMEM_ALIGN_T uiBlockSize, |
| IMG_UINT32 uiLog2Contiguity, |
| IMG_HANDLE hPDumpAllocationInfoHandle) |
| { |
| PVRSRV_ERROR eError; |
| IMG_UINT32 i; |
| |
| /* (IMG_HANDLE*) <- (IMG_HANDLE) */ |
| IMG_HANDLE *ahPDumpAllocHandleArray = (IMG_HANDLE*) hPDumpAllocationInfoHandle; |
| |
| for (i = 0; i < psPMR->uiNumPDumpBlocks; i++) |
| { |
| if (ahPDumpAllocHandleArray[i] != NULL) |
| { |
| eError = PDumpFree(ahPDumpAllocHandleArray[i]); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| ahPDumpAllocHandleArray[i] = NULL; |
| } |
| } |
| |
| OSFreeMem(ahPDumpAllocHandleArray); |
| } |
| |
| |
| void |
| PDumpPMRMallocPMR(PMR *psPMR, |
| IMG_DEVMEM_SIZE_T uiSize, |
| IMG_DEVMEM_ALIGN_T uiBlockSize, |
| IMG_UINT32 ui32ChunkSize, |
| IMG_UINT32 ui32NumPhysChunks, |
| IMG_UINT32 ui32NumVirtChunks, |
| IMG_UINT32 *puiMappingTable, |
| IMG_UINT32 uiLog2Contiguity, |
| IMG_BOOL bInitialise, |
| IMG_UINT32 ui32InitValue, |
| IMG_HANDLE *phPDumpAllocInfoOut, |
| IMG_UINT32 ui32PDumpFlags) |
| { |
| PVRSRV_ERROR eError; |
| IMG_HANDLE *phPDumpAllocInfo; |
| |
| IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName; |
| IMG_UINT32 uiNumPhysBlocks; |
| IMG_UINT32 uiNumVirtBlocks; |
| IMG_UINT32 i, uiIndex; |
| |
| |
| if (PMR_IsSparse(psPMR)) |
| { |
| uiNumPhysBlocks = (ui32ChunkSize * ui32NumPhysChunks) >> uiLog2Contiguity; |
| /* Make sure we did not cut off anything */ |
| PVR_ASSERT(uiNumPhysBlocks << uiLog2Contiguity == (ui32ChunkSize * ui32NumPhysChunks)); |
| } |
| else |
| { |
| uiNumPhysBlocks = uiSize >> uiLog2Contiguity; |
| /* Make sure we did not cut off anything */ |
| PVR_ASSERT(uiNumPhysBlocks << uiLog2Contiguity == uiSize); |
| } |
| |
| uiNumVirtBlocks = uiSize >> uiLog2Contiguity; |
| PVR_ASSERT(uiNumVirtBlocks << uiLog2Contiguity == uiSize); |
| |
| psPMR->uiNumPDumpBlocks = uiNumVirtBlocks; |
| |
| phPDumpAllocInfo = (IMG_HANDLE*) OSAllocZMem(uiNumVirtBlocks * sizeof(IMG_HANDLE)); |
| |
| |
| for (i = 0; i < uiNumPhysBlocks; i++) |
| { |
| uiIndex = PMR_IsSparse(psPMR) ? puiMappingTable[i] : i; |
| |
| eError = PMR_PDumpSymbolicAddr(psPMR, |
| uiIndex * uiBlockSize, |
| sizeof(aszMemspaceName), |
| &aszMemspaceName[0], |
| sizeof(aszSymbolicName), |
| &aszSymbolicName[0], |
| &uiOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| eError = PDumpMalloc(aszMemspaceName, |
| aszSymbolicName, |
| uiBlockSize, |
| uiBlockSize, |
| bInitialise, |
| ui32InitValue, |
| &phPDumpAllocInfo[uiIndex], |
| ui32PDumpFlags); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| } |
| |
| /* (IMG_HANDLE) <- (IMG_HANDLE*) */ |
| *phPDumpAllocInfoOut = (IMG_HANDLE) phPDumpAllocInfo; |
| |
| } |
| #endif /* PDUMP */ |
| |
| |
| void *PMRGetPrivateData(const PMR *psPMR, |
| const PMR_IMPL_FUNCTAB *psFuncTab) |
| { |
| return (psFuncTab == psPMR->psFuncTab) ? psPMR->pvFlavourData : NULL; |
| } |
| |
| #define PMR_PM_WORD_SIZE 4 |
| |
| PVRSRV_ERROR |
| PMRWritePMPageList(/* Target PMR, offset, and length */ |
| PMR *psPageListPMR, |
| IMG_DEVMEM_OFFSET_T uiTableOffset, |
| IMG_DEVMEM_SIZE_T uiTableLength, |
| /* Referenced PMR, and "page" granularity */ |
| PMR *psReferencePMR, |
| IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize, |
| PMR_PAGELIST **ppsPageList) |
| { |
| PVRSRV_ERROR eError; |
| IMG_DEVMEM_SIZE_T uiWordSize; |
| IMG_UINT32 uiNumPages; |
| IMG_UINT32 uiPageIndex; |
| PMR_FLAGS_T uiFlags = psPageListPMR->uiFlags; |
| PMR_PAGELIST *psPageList; |
| #if defined(PDUMP) |
| IMG_CHAR aszTableEntryMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszTableEntrySymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiTableEntryPDumpOffset; |
| IMG_CHAR aszPageMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH]; |
| IMG_CHAR aszPageSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH]; |
| IMG_DEVMEM_OFFSET_T uiPagePDumpOffset; |
| IMG_DEVMEM_OFFSET_T uiNextSymName; |
| #endif |
| #if !defined(NO_HARDWARE) |
| IMG_UINT32 uiPageListPageSize = 1 << psPageListPMR->uiLog2ContiguityGuarantee; |
| IMG_UINT64 uiPageListPMRPage = 0; |
| IMG_UINT64 uiPrevPageListPMRPage = 0; |
| IMG_HANDLE hPrivData = NULL; |
| void *pvKernAddr = NULL; |
| IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC]; |
| IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC]; |
| IMG_DEV_PHYADDR *pasDevAddrPtr; |
| IMG_UINT32 *pui32DataPtr = NULL; |
| IMG_BOOL *pbPageIsValid; |
| #endif |
| |
| uiWordSize = PMR_PM_WORD_SIZE; |
| |
| /* check we're being asked to write the same number of 4-byte units as there are pages */ |
| uiNumPages = (IMG_UINT32)(psReferencePMR->uiLogicalSize >> uiLog2PageSize); |
| |
| if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psReferencePMR->uiLogicalSize) |
| { |
| /* Strictly speaking, it's possible to provoke this error in two ways: |
| (i) if it's not a whole multiple of the page size; or |
| (ii) if there are more than 4 billion pages. |
| The latter is unlikely. :) but the check is required in order to justify the cast. |
| */ |
| eError = PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE; |
| goto return_error; |
| } |
| uiWordSize = (IMG_UINT32)uiTableLength / uiNumPages; |
| if (uiNumPages * uiWordSize != uiTableLength) |
| { |
| eError = PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE; |
| goto return_error; |
| } |
| |
| /* Check we're not being asked to write off the end of the PMR */ |
| if (uiTableOffset + uiTableLength > psPageListPMR->uiLogicalSize) |
| { |
| /* table memory insufficient to store all the entries */ |
| /* table insufficient to store addresses of whole block */ |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto return_error; |
| } |
| |
| /* the PMR into which we are writing must not be user CPU mappable: */ |
| if (PVRSRV_CHECK_CPU_READABLE(uiFlags) || PVRSRV_CHECK_CPU_WRITEABLE(uiFlags)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, |
| "Masked flags = 0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC, |
| (uiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE | PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE)))); |
| PVR_DPF((PVR_DBG_ERROR, |
| "Page list PMR allows CPU mapping (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")", |
| uiFlags)); |
| eError = PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS; |
| goto return_error; |
| } |
| |
| if (_PMRIsSparse(psPageListPMR)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "PageList PMR is sparse")); |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto return_error; |
| } |
| |
| if (_PMRIsSparse(psReferencePMR)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "Reference PMR is sparse")); |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto return_error; |
| } |
| |
| psPageList = OSAllocMem(sizeof(PMR_PAGELIST)); |
| if (psPageList == NULL) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "Failed to allocate PMR page list")); |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto return_error; |
| } |
| psPageList->psReferencePMR = psReferencePMR; |
| |
| /* Need to lock down the physical addresses of the reference PMR */ |
| /* N.B. This also checks that the requested "contiguity" is achievable */ |
| eError = PMRLockSysPhysAddresses(psReferencePMR); |
| if (eError != PVRSRV_OK) |
| { |
| goto free_page_list; |
| } |
| |
| #if !defined(NO_HARDWARE) |
| if (uiNumPages > PMR_MAX_TRANSLATION_STACK_ALLOC) |
| { |
| pasDevAddrPtr = OSAllocMem(uiNumPages * sizeof(IMG_DEV_PHYADDR)); |
| if (pasDevAddrPtr == NULL) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "Failed to allocate PMR page list")); |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto unlock_phys_addrs; |
| } |
| |
| pbPageIsValid = OSAllocMem(uiNumPages * sizeof(IMG_BOOL)); |
| if (pbPageIsValid == NULL) |
| { |
| /* Clean-up before exit */ |
| OSFreeMem(pasDevAddrPtr); |
| |
| PVR_DPF((PVR_DBG_ERROR, "Failed to allocate PMR page state")); |
| eError = PVRSRV_ERROR_OUT_OF_MEMORY; |
| goto free_devaddr_array; |
| } |
| } |
| else |
| { |
| pasDevAddrPtr = asDevPAddr; |
| pbPageIsValid = abValid; |
| } |
| |
| eError = PMR_DevPhysAddr(psReferencePMR, uiLog2PageSize, uiNumPages, 0, |
| pasDevAddrPtr, pbPageIsValid); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "Failed to map PMR pages into device physical addresses")); |
| goto free_valid_array; |
| } |
| #endif |
| |
| for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++) |
| { |
| IMG_DEVMEM_OFFSET_T uiPMROffset = uiTableOffset + (uiWordSize * uiPageIndex); |
| #if defined(PDUMP) |
| eError = PMR_PDumpSymbolicAddr(psPageListPMR, |
| uiPMROffset, |
| sizeof(aszTableEntryMemspaceName), |
| &aszTableEntryMemspaceName[0], |
| sizeof(aszTableEntrySymbolicName), |
| &aszTableEntrySymbolicName[0], |
| &uiTableEntryPDumpOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| eError = PMR_PDumpSymbolicAddr(psReferencePMR, |
| (IMG_DEVMEM_OFFSET_T)uiPageIndex << uiLog2PageSize, |
| sizeof(aszPageMemspaceName), |
| &aszPageMemspaceName[0], |
| sizeof(aszPageSymbolicName), |
| &aszPageSymbolicName[0], |
| &uiPagePDumpOffset, |
| &uiNextSymName); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| |
| eError = PDumpWriteShiftedMaskedValue(/* destination */ |
| aszTableEntryMemspaceName, |
| aszTableEntrySymbolicName, |
| uiTableEntryPDumpOffset, |
| /* source */ |
| aszPageMemspaceName, |
| aszPageSymbolicName, |
| uiPagePDumpOffset, |
| /* shift right */ |
| uiLog2PageSize, |
| /* shift left */ |
| 0, |
| /* mask */ |
| 0xffffffff, |
| /* word size */ |
| uiWordSize, |
| /* flags */ |
| PDUMP_FLAGS_CONTINUOUS); |
| PVR_ASSERT(eError == PVRSRV_OK); |
| #else |
| PVR_UNREFERENCED_PARAMETER(uiPMROffset); |
| #endif |
| #if !defined(NO_HARDWARE) |
| |
| /* |
| We check for sparse PMR's at function entry, but as we can, |
| check that every page is valid |
| */ |
| PVR_ASSERT(pbPageIsValid[uiPageIndex]); |
| PVR_ASSERT(pasDevAddrPtr[uiPageIndex].uiAddr != 0); |
| PVR_ASSERT(((pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize) & 0xFFFFFFFF00000000ll) == 0); |
| |
| uiPageListPMRPage = uiPMROffset >> psReferencePMR->uiLog2ContiguityGuarantee; |
| |
| if ((pui32DataPtr == NULL) || (uiPageListPMRPage != uiPrevPageListPMRPage)) |
| { |
| size_t uiMappingOffset = uiPMROffset & (~(uiPageListPageSize - 1)); |
| size_t uiMappedSize; |
| |
| /* If we already had a page list mapped, we need to unmap it... */ |
| if (pui32DataPtr != NULL) |
| { |
| PMRReleaseKernelMappingData(psPageListPMR, hPrivData); |
| } |
| |
| eError = PMRAcquireKernelMappingData(psPageListPMR, |
| uiMappingOffset, |
| uiPageListPageSize, |
| &pvKernAddr, |
| &uiMappedSize, |
| &hPrivData); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "Error mapping page list PMR page (%" IMG_UINT64_FMTSPEC ") into kernel (%d)", |
| uiPageListPMRPage, eError)); |
| goto free_valid_array; |
| } |
| |
| uiPrevPageListPMRPage = uiPageListPMRPage; |
| PVR_ASSERT(uiMappedSize >= uiPageListPageSize); |
| PVR_ASSERT(pvKernAddr != NULL); |
| |
| pui32DataPtr = (IMG_UINT32 *) (((IMG_CHAR *) pvKernAddr) + (uiPMROffset & (uiPageListPageSize - 1))); |
| } |
| |
| PVR_ASSERT(((pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize) & 0xFFFFFFFF00000000ll) == 0); |
| |
| /* Write the physical page index into the page list PMR */ |
| *pui32DataPtr++ = TRUNCATE_64BITS_TO_32BITS(pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize); |
| |
| /* Last page so unmap */ |
| if (uiPageIndex == (uiNumPages - 1)) |
| { |
| PMRReleaseKernelMappingData(psPageListPMR, hPrivData); |
| } |
| #endif |
| } |
| |
| #if !defined(NO_HARDWARE) |
| if (pasDevAddrPtr != asDevPAddr) |
| { |
| OSFreeMem(pbPageIsValid); |
| OSFreeMem(pasDevAddrPtr); |
| } |
| #endif |
| *ppsPageList = psPageList; |
| return PVRSRV_OK; |
| |
| /* |
| error exit paths follow |
| */ |
| #if !defined(NO_HARDWARE) |
| |
| free_valid_array: |
| if (pbPageIsValid != abValid) |
| { |
| OSFreeMem(pbPageIsValid); |
| } |
| |
| free_devaddr_array: |
| if (pasDevAddrPtr != asDevPAddr) |
| { |
| OSFreeMem(pasDevAddrPtr); |
| } |
| |
| unlock_phys_addrs: |
| PMRUnlockSysPhysAddresses(psReferencePMR); |
| #endif |
| |
| free_page_list: |
| OSFreeMem(psPageList); |
| |
| return_error: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| |
| PVRSRV_ERROR /* FIXME: should be void */ |
| PMRUnwritePMPageList(PMR_PAGELIST *psPageList) |
| { |
| PVRSRV_ERROR eError2; |
| |
| eError2 = PMRUnlockSysPhysAddresses(psPageList->psReferencePMR); |
| PVR_ASSERT(eError2 == PVRSRV_OK); |
| OSFreeMem(psPageList); |
| |
| return PVRSRV_OK; |
| } |
| |
| PVRSRV_ERROR |
| PMRZeroingPMR(PMR *psPMR, |
| IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize) |
| { |
| IMG_UINT32 uiNumPages; |
| IMG_UINT32 uiPageIndex; |
| IMG_UINT32 ui32PageSize = 1 << uiLog2PageSize; |
| IMG_HANDLE hPrivData = NULL; |
| void *pvKernAddr = NULL; |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| size_t uiMapedSize; |
| |
| PVR_ASSERT(psPMR); |
| |
| /* Calculate number of pages in this PMR */ |
| uiNumPages = (IMG_UINT32)(psPMR->uiLogicalSize >> uiLog2PageSize); |
| |
| /* Verify the logical Size is a multiple or the physical page size */ |
| if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psPMR->uiLogicalSize) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "PMRZeroingPMR: PMR is not a multiple of %u",ui32PageSize)); |
| eError = PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE; |
| goto MultiPage_Error; |
| } |
| |
| if (_PMRIsSparse(psPMR)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "PMRZeroingPMR: PMR is sparse")); |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto Sparse_Error; |
| } |
| |
| /* Scan through all pages of the PMR */ |
| for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++) |
| { |
| /* map the physical page (for a given PMR offset) into kernel space */ |
| eError = PMRAcquireKernelMappingData(psPMR, |
| (size_t)uiPageIndex << uiLog2PageSize, |
| ui32PageSize, |
| &pvKernAddr, |
| &uiMapedSize, |
| &hPrivData); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "PMRZeroingPMR: AcquireKernelMapping failed with error %u", eError)); |
| goto AcquireKernelMapping_Error; |
| } |
| |
| /* ensure the mapped page size is the same as the physical page size */ |
| if (uiMapedSize != ui32PageSize) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "PMRZeroingPMR: Physical Page size = 0x%08x, Size of Mapping = 0x%016" IMG_UINT64_FMTSPECx, |
| ui32PageSize, |
| (IMG_UINT64)uiMapedSize)); |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto MappingSize_Error; |
| } |
| |
| /* Use the conservative 'DeviceMemSet' here because we can't know |
| * if this PMR will be mapped cached. |
| */ |
| |
| OSDeviceMemSet(pvKernAddr, 0, ui32PageSize); |
| |
| /* release mapping */ |
| PMRReleaseKernelMappingData(psPMR, hPrivData); |
| |
| } |
| |
| PVR_DPF((PVR_DBG_MESSAGE,"PMRZeroingPMR: Zeroing PMR %p done (num pages %u, page size %u)", |
| psPMR, |
| uiNumPages, |
| ui32PageSize)); |
| |
| return PVRSRV_OK; |
| |
| |
| /* Error handling */ |
| |
| MappingSize_Error: |
| PMRReleaseKernelMappingData(psPMR, hPrivData); |
| |
| AcquireKernelMapping_Error: |
| Sparse_Error: |
| MultiPage_Error: |
| |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRDumpPageList(PMR *psPMR, |
| IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize) |
| { |
| IMG_DEV_PHYADDR sDevAddrPtr; |
| IMG_UINT32 uiNumPages; |
| IMG_UINT32 uiPageIndex; |
| IMG_BOOL bPageIsValid; |
| IMG_UINT32 ui32Col = 16; |
| IMG_UINT32 ui32SizePerCol = 11; |
| IMG_UINT32 ui32ByteCount = 0; |
| IMG_CHAR pszBuffer[16 /* ui32Col */ * 11 /* ui32SizePerCol */ + 1]; |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| |
| /* Get number of pages */ |
| uiNumPages = (IMG_UINT32)(psPMR->uiLogicalSize >> uiLog2PageSize); |
| |
| /* Verify the logical Size is a multiple or the physical page size */ |
| if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psPMR->uiLogicalSize) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "PMRPrintPageList: PMR is not a multiple of %u", 1 << uiLog2PageSize)); |
| eError = PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE; |
| goto MultiPage_Error; |
| } |
| |
| if (_PMRIsSparse(psPMR)) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "PMRPrintPageList: PMR is sparse")); |
| eError = PVRSRV_ERROR_INVALID_PARAMS; |
| goto Sparse_Error; |
| } |
| |
| PVR_LOG((" PMR %p, Number of pages %u, Log2PageSize %d", psPMR, uiNumPages, uiLog2PageSize)); |
| |
| /* Print the address of the physical pages */ |
| for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++) |
| { |
| /* Get Device physical Address */ |
| eError = PMR_DevPhysAddr(psPMR, |
| uiLog2PageSize, |
| 1, |
| (IMG_DEVMEM_OFFSET_T)uiPageIndex << uiLog2PageSize, |
| &sDevAddrPtr, |
| &bPageIsValid); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "PMRPrintPageList: PMR %p failed to get DevPhysAddr with error %u", |
| psPMR, |
| eError)); |
| goto DevPhysAddr_Error; |
| } |
| |
| ui32ByteCount += OSSNPrintf(pszBuffer + ui32ByteCount, ui32SizePerCol + 1, "%08x ", (IMG_UINT32)(sDevAddrPtr.uiAddr >> uiLog2PageSize)); |
| PVR_ASSERT(ui32ByteCount < ui32Col * ui32SizePerCol); |
| |
| if (uiPageIndex % ui32Col == ui32Col-1) |
| { |
| PVR_LOG((" Phys Page: %s", pszBuffer)); |
| ui32ByteCount = 0; |
| } |
| } |
| if (ui32ByteCount > 0) |
| { |
| PVR_LOG((" Phys Page: %s", pszBuffer)); |
| } |
| |
| return PVRSRV_OK; |
| |
| /* Error handling */ |
| DevPhysAddr_Error: |
| Sparse_Error: |
| MultiPage_Error: |
| PVR_ASSERT(eError != PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRInit(void) |
| { |
| PVRSRV_ERROR eError; |
| |
| if (_gsSingletonPMRContext.bModuleInitialised) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Error: Singleton PMR context already initialized", __func__)); |
| eError = PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR; |
| goto out; |
| } |
| |
| eError = OSLockCreate(&_gsSingletonPMRContext.hLock); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Error: failed to create lock", __func__)); |
| goto out; |
| } |
| |
| _gsSingletonPMRContext.uiNextSerialNum = 1; |
| |
| _gsSingletonPMRContext.uiNextKey = 0x8300f001 * (uintptr_t)&_gsSingletonPMRContext; |
| |
| _gsSingletonPMRContext.bModuleInitialised = IMG_TRUE; |
| |
| _gsSingletonPMRContext.uiNumLivePMRs = 0; |
| |
| #if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS) |
| eError = MMapStatsInit(); |
| if (eError != PVRSRV_OK) |
| { |
| PVR_DPF((PVR_DBG_ERROR,"%s: MMap stats initialisation failed", __func__)); |
| goto out; |
| } |
| #endif |
| |
| out: |
| PVR_ASSERT(eError == PVRSRV_OK); |
| return eError; |
| } |
| |
| PVRSRV_ERROR |
| PMRDeInit(void) |
| { |
| PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData(); |
| PVRSRV_ERROR eError = PVRSRV_OK; |
| |
| if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK) |
| { |
| goto out; |
| } |
| |
| if (!_gsSingletonPMRContext.bModuleInitialised) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Error: Singleton PMR context is not initialized", __func__)); |
| eError = PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR; |
| goto out; |
| } |
| |
| #if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS) |
| MMapStatsDeInit(); |
| #endif |
| |
| if (_gsSingletonPMRContext.uiNumLivePMRs != 0) |
| { |
| PVR_DPF((PVR_DBG_ERROR, "%s: Error: %d live PMRs remain", |
| __func__, |
| _gsSingletonPMRContext.uiNumLivePMRs)); |
| PVR_DPF((PVR_DBG_ERROR, "%s: This is an unrecoverable error; a subsequent crash is inevitable", |
| __func__)); |
| eError = PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR; |
| goto out; |
| } |
| |
| OSLockDestroy(_gsSingletonPMRContext.hLock); |
| |
| _gsSingletonPMRContext.bModuleInitialised = IMG_FALSE; |
| |
| /* |
| FIXME: |
| |
| should deinitialise the mutex here |
| */ |
| out: |
| PVR_ASSERT(eError == PVRSRV_OK); |
| return eError; |
| } |