| /**************************************************************************** |
| * |
| * The MIT License (MIT) |
| * |
| * Copyright (c) 2014 - 2020 Vivante Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in |
| * all copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| * DEALINGS IN THE SOFTWARE. |
| * |
| ***************************************************************************** |
| * |
| * The GPL License (GPL) |
| * |
| * Copyright (C) 2014 - 2020 Vivante Corporation |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; either version 2 |
| * of the License, or (at your option) any later version. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| * |
| ***************************************************************************** |
| * |
| * Note: This software is released under dual MIT and GPL licenses. A |
| * recipient may use this file under the terms of either the MIT license or |
| * GPL License. If you wish to use only one license not the other, you can |
| * indicate your decision by deleting one of the above license notices in your |
| * version of this file. |
| * |
| *****************************************************************************/ |
| |
| |
| #include "gc_hal_kernel_precomp.h" |
| |
| #define _GC_OBJ_ZONE gcvZONE_MMU |
| |
| typedef enum _gceMMU_TYPE |
| { |
| gcvMMU_USED = (0 << 4), |
| gcvMMU_SINGLE = (1 << 4), |
| gcvMMU_FREE = (2 << 4), |
| } |
| gceMMU_TYPE; |
| |
| /* VIP SRAM start virtual address. */ |
| #define gcdRESERVE_START (4 << 20) |
| #define gcdRESERVE_ALIGN (4 << 10) |
| |
| #define gcmENTRY_TYPE(x) (x & 0xF0) |
| |
| #define gcmENTRY_COUNT(x) ((x & 0xFFFFFF00) >> 8) |
| |
| #define gcdMMU_TABLE_DUMP 0 |
| |
| #define gcdVERTEX_START (128 << 10) |
| |
| typedef struct _gcsMMU_STLB_CHUNK *gcsMMU_STLB_CHUNK_PTR; |
| |
| typedef struct _gcsMMU_STLB_CHUNK |
| { |
| gckVIDMEM_NODE videoMem; |
| gctUINT32_PTR logical; |
| gctSIZE_T size; |
| gctPHYS_ADDR_T physBase; |
| gctSIZE_T pageCount; |
| gctUINT32 mtlbIndex; |
| gctUINT32 mtlbEntryNum; |
| gcsMMU_STLB_CHUNK_PTR next; |
| } gcsMMU_STLB_CHUNK; |
| |
| #if gcdSHARED_PAGETABLE |
| typedef struct _gcsSharedPageTable * gcsSharedPageTable_PTR; |
| typedef struct _gcsSharedPageTable |
| { |
| /* Shared gckMMU object. */ |
| gckMMU mmu; |
| |
| /* Hardwares which use this shared pagetable. */ |
| gckHARDWARE hardwares[gcdMAX_GPU_COUNT]; |
| |
| /* Number of cores use this shared pagetable. */ |
| gctUINT32 reference; |
| } |
| gcsSharedPageTable; |
| |
| static gcsSharedPageTable_PTR sharedPageTable = gcvNULL; |
| #endif |
| |
| typedef struct _gcsFreeSpaceNode * gcsFreeSpaceNode_PTR; |
| typedef struct _gcsFreeSpaceNode |
| { |
| gctUINT32 start; |
| gctINT32 entries; |
| } |
| gcsFreeSpaceNode; |
| |
| #if gcdENDIAN_BIG |
| |
| # define _WritePageEntry(pageEntry, entryValue) \ |
| *(gctUINT32_PTR)(pageEntry) = gcmBSWAP32((gctUINT32)(entryValue)) |
| |
| # define _ReadPageEntry(pageEntry) \ |
| gcmBSWAP32(*(gctUINT32_PTR)(pageEntry)) |
| |
| #else |
| |
| # define _WritePageEntry(pageEntry, entryValue) \ |
| *(gctUINT32_PTR)(pageEntry) = (gctUINT32)(entryValue) |
| |
| # define _ReadPageEntry(pageEntry) \ |
| *(gctUINT32_PTR)(pageEntry) |
| |
| #endif |
| |
| static gceSTATUS |
| _FillPageTable( |
| IN gctUINT32_PTR PageTable, |
| IN gctUINT32 PageCount, |
| IN gctUINT32 EntryValue |
| ) |
| { |
| gctUINT i; |
| |
| for (i = 0; i < PageCount; i++) |
| { |
| _WritePageEntry(PageTable + i, EntryValue); |
| } |
| |
| return gcvSTATUS_OK; |
| } |
| |
| static gceSTATUS |
| _FillMap( |
| IN gctUINT32_PTR Map, |
| IN gctUINT32 PageCount, |
| IN gctUINT32 EntryValue |
| ) |
| { |
| gctUINT i; |
| |
| for (i = 0; i < PageCount; i++) |
| { |
| Map[i] = EntryValue; |
| } |
| |
| return gcvSTATUS_OK; |
| } |
| |
| static gceSTATUS |
| _Link( |
| IN gcsADDRESS_AREA_PTR Area, |
| IN gctUINT32 Index, |
| IN gctUINT32 Next |
| ) |
| { |
| if (Index >= Area->stlbEntries) |
| { |
| /* Just move heap pointer. */ |
| Area->heapList = Next; |
| } |
| else |
| { |
| /* Address page table. */ |
| gctUINT32_PTR map = Area->mapLogical; |
| |
| /* Dispatch on node type. */ |
| switch (gcmENTRY_TYPE(map[Index])) |
| { |
| case gcvMMU_SINGLE: |
| /* Set single index. */ |
| map[Index] = (Next << 8) | gcvMMU_SINGLE; |
| break; |
| |
| case gcvMMU_FREE: |
| /* Set index. */ |
| map[Index + 1] = Next; |
| break; |
| |
| default: |
| gcmkFATAL("MMU table correcupted at index %u!", Index); |
| return gcvSTATUS_HEAP_CORRUPTED; |
| } |
| } |
| |
| /* Success. */ |
| return gcvSTATUS_OK; |
| } |
| |
| static gceSTATUS |
| _AddFree( |
| IN gcsADDRESS_AREA_PTR Area, |
| IN gctUINT32 Index, |
| IN gctUINT32 Node, |
| IN gctUINT32 Count |
| ) |
| { |
| gctUINT32_PTR map = Area->mapLogical; |
| |
| if (Count == 1) |
| { |
| /* Initialize a single page node. */ |
| map[Node] = (~((1U<<8)-1)) | gcvMMU_SINGLE; |
| } |
| else |
| { |
| /* Initialize the node. */ |
| map[Node + 0] = (Count << 8) | gcvMMU_FREE; |
| map[Node + 1] = ~0U; |
| } |
| |
| /* Append the node. */ |
| return _Link(Area, Index, Node); |
| } |
| |
| static gceSTATUS |
| _Collect( |
| IN gcsADDRESS_AREA_PTR Area |
| ) |
| { |
| gctUINT32_PTR map = Area->mapLogical; |
| gceSTATUS status; |
| gctUINT32 i, previous, start = 0, count = 0; |
| |
| previous = Area->heapList = ~0U; |
| Area->freeNodes = gcvFALSE; |
| |
| /* Walk the entire page table. */ |
| for (i = 0; i < Area->stlbEntries; ++i) |
| { |
| /* Dispatch based on type of page. */ |
| switch (gcmENTRY_TYPE(map[i])) |
| { |
| case gcvMMU_USED: |
| /* Used page, so close any open node. */ |
| if (count > 0) |
| { |
| /* Add the node. */ |
| gcmkONERROR(_AddFree(Area, previous, start, count)); |
| |
| /* Reset the node. */ |
| previous = start; |
| count = 0; |
| } |
| break; |
| |
| case gcvMMU_SINGLE: |
| /* Single free node. */ |
| if (count++ == 0) |
| { |
| /* Start a new node. */ |
| start = i; |
| } |
| break; |
| |
| case gcvMMU_FREE: |
| /* A free node. */ |
| if (count == 0) |
| { |
| /* Start a new node. */ |
| start = i; |
| } |
| |
| /* Advance the count. */ |
| count += map[i] >> 8; |
| |
| /* Advance the index into the page table. */ |
| i += (map[i] >> 8) - 1; |
| break; |
| |
| default: |
| gcmkFATAL("MMU page table correcupted at index %u!", i); |
| return gcvSTATUS_HEAP_CORRUPTED; |
| } |
| } |
| |
| /* See if we have an open node left. */ |
| if (count > 0) |
| { |
| /* Add the node to the list. */ |
| gcmkONERROR(_AddFree(Area, previous, start, count)); |
| } |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_MMU, |
| "Performed a garbage collection of the MMU heap."); |
| |
| /* Success. */ |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the staus. */ |
| return status; |
| } |
| |
| static gctUINT32 |
| _SetPage(gctUINT32 PageAddress, gctUINT32 PageAddressExt, gctBOOL Writable) |
| { |
| gctUINT32 entry = PageAddress |
| /* AddressExt */ |
| | (PageAddressExt << 4) |
| /* Ignore exception */ |
| | (0 << 1) |
| /* Present */ |
| | (1 << 0); |
| |
| if (Writable) |
| { |
| /* writable */ |
| entry |= (1 << 2); |
| } |
| #if gcdUSE_MMU_EXCEPTION |
| else |
| { |
| /* If this page is read only, set exception bit to make exception happens |
| ** when writing to it. */ |
| entry |= gcdMMU_STLB_EXCEPTION; |
| } |
| #endif |
| |
| return entry; |
| } |
| |
| static gctUINT32 |
| _MtlbOffset( |
| gctUINT32 Address |
| ) |
| { |
| return (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; |
| } |
| |
| gctUINT32 |
| _AddressToIndex( |
| IN gcsADDRESS_AREA_PTR Area, |
| IN gctUINT32 Address |
| ) |
| { |
| |
| gctUINT32 stlbShift = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT; |
| gctUINT32 stlbMask = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_MASK : gcdMMU_STLB_4K_MASK; |
| gctUINT32 stlbEntryNum = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM; |
| gctUINT32 mtlbOffset = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; |
| gctUINT32 stlbOffset = (Address & stlbMask) >> stlbShift; |
| |
| return (mtlbOffset - Area->mappingStart) * stlbEntryNum + stlbOffset; |
| } |
| |
| static gctUINT32_PTR |
| _StlbEntry( |
| gcsADDRESS_AREA_PTR Area, |
| gctUINT32 Address |
| ) |
| { |
| gctUINT32 index = _AddressToIndex(Area, Address); |
| |
| return &Area->stlbLogical[index]; |
| } |
| |
| static gctBOOL |
| _IsRangeInsected( |
| gctUINT64 baseAddress1, |
| gctSIZE_T size1, |
| gctUINT64 baseAddress2, |
| gctSIZE_T size2 |
| ) |
| { |
| gctUINT64 endAddress1 = baseAddress1 + size1 - 1; |
| gctUINT64 endAddress2 = baseAddress2 + size2 - 1; |
| |
| if (!size1 || !size2) |
| { |
| return gcvFALSE; |
| } |
| |
| return (((baseAddress2 <= endAddress1) && (endAddress2 >= baseAddress1)) || |
| ((baseAddress1 <= endAddress2) && (endAddress1 >= baseAddress2))); |
| } |
| |
| static gceSTATUS |
| _FillFlatMappingInMap( |
| gcsADDRESS_AREA_PTR Area, |
| gctUINT32 Index, |
| gctUINT32 NumPages |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 i; |
| gctBOOL gotIt = gcvFALSE; |
| gctUINT32 index = Index; |
| gctUINT32_PTR map = Area->mapLogical; |
| gctUINT32 previous = ~0U; |
| |
| /* Find node which contains index. */ |
| for (i = 0; !gotIt && (i < Area->stlbEntries);) |
| { |
| gctUINT32 numPages; |
| |
| switch (gcmENTRY_TYPE(map[i])) |
| { |
| case gcvMMU_SINGLE: |
| if (i == index) |
| { |
| gotIt = gcvTRUE; |
| } |
| else |
| { |
| previous = i; |
| i = map[i] >> 8; |
| } |
| break; |
| |
| case gcvMMU_FREE: |
| numPages = map[i] >> 8; |
| if (index >= i && index + NumPages - 1 < i + numPages) |
| { |
| gotIt = gcvTRUE; |
| } |
| else |
| { |
| previous = i; |
| i = map[i + 1]; |
| } |
| break; |
| |
| case gcvMMU_USED: |
| i++; |
| break; |
| |
| default: |
| gcmkFATAL("MMU table correcupted at index %u!", index); |
| gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); |
| } |
| } |
| |
| switch (gcmENTRY_TYPE(map[i])) |
| { |
| case gcvMMU_SINGLE: |
| /* Unlink single node from free list. */ |
| gcmkONERROR( |
| _Link(Area, previous, map[i] >> 8)); |
| break; |
| |
| case gcvMMU_FREE: |
| /* Split the node. */ |
| { |
| gctUINT32 start; |
| gctUINT32 next = map[i+1]; |
| gctUINT32 total = map[i] >> 8; |
| gctUINT32 countLeft = index - i; |
| gctUINT32 countRight = total - countLeft - NumPages; |
| |
| if (countLeft) |
| { |
| start = i; |
| _AddFree(Area, previous, start, countLeft); |
| previous = start; |
| } |
| |
| if (countRight) |
| { |
| start = index + NumPages; |
| _AddFree(Area, previous, start, countRight); |
| previous = start; |
| } |
| |
| _Link(Area, previous, next); |
| } |
| break; |
| } |
| |
| _FillMap(&map[index], NumPages, gcvMMU_USED); |
| |
| return gcvSTATUS_OK; |
| OnError: |
| return status; |
| } |
| |
| static gceSTATUS |
| _CollectFreeSpace( |
| IN gckMMU Mmu, |
| OUT gcsFreeSpaceNode_PTR *Array, |
| OUT gctINT * Size |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gctPOINTER pointer = gcvNULL; |
| gcsFreeSpaceNode_PTR array = gcvNULL; |
| gcsFreeSpaceNode_PTR node = gcvNULL; |
| gctINT size = 0; |
| gctINT i = 0; |
| |
| for (i = 0; i < gcdMMU_MTLB_ENTRY_NUM; i++) |
| { |
| if (!Mmu->mtlbLogical[i]) |
| { |
| if (!node) |
| { |
| /* This is the first entry of the free space. */ |
| node += 1; |
| size++; |
| |
| } |
| } |
| else if (node) |
| { |
| /* Reset the start. */ |
| node = gcvNULL; |
| } |
| } |
| |
| /* Allocate memory for the array. */ |
| gcmkONERROR(gckOS_Allocate(Mmu->os, |
| gcmSIZEOF(*array) * size, |
| &pointer)); |
| |
| array = (gcsFreeSpaceNode_PTR)pointer; |
| node = gcvNULL; |
| |
| for (i = 0, size = 0; i < gcdMMU_MTLB_ENTRY_NUM; i++) |
| { |
| if (!Mmu->mtlbLogical[i]) |
| { |
| if (!node) |
| { |
| /* This is the first entry of the free space. */ |
| node = &array[size++]; |
| |
| node->start = i; |
| node->entries = 0; |
| } |
| |
| node->entries++; |
| } |
| else if (node) |
| { |
| /* Reset the start. */ |
| node = gcvNULL; |
| } |
| } |
| |
| #if gcdMMU_TABLE_DUMP |
| for (i = 0; i < size; i++) |
| { |
| gckOS_Print("%s(%d): [%d]: start=%d, entries=%d.\n", |
| __FUNCTION__, __LINE__, |
| i, |
| array[i].start, |
| array[i].entries); |
| } |
| #endif |
| |
| *Array = array; |
| *Size = size; |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (pointer != gcvNULL) |
| { |
| gckOS_Free(Mmu->os, pointer); |
| } |
| |
| return status; |
| } |
| |
| gceSTATUS |
| _GetMtlbFreeSpace( |
| IN gckMMU Mmu, |
| IN gctUINT32 NumEntries, |
| OUT gctUINT32 *MtlbStart, |
| OUT gctUINT32 *MtlbEnd |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gcsFreeSpaceNode_PTR nodeArray = gcvNULL; |
| gctINT i, nodeArraySize = 0; |
| gctINT numEntries = gcdMMU_MTLB_ENTRY_NUM; |
| gctINT32 mStart = -1; |
| gctINT32 mEnd = -1; |
| |
| gcmkONERROR(_CollectFreeSpace(Mmu, &nodeArray, &nodeArraySize)); |
| |
| /* Find the smallest space for NumEntries */ |
| for (i = 0; i < nodeArraySize; i++) |
| { |
| if (nodeArray[i].entries < numEntries && NumEntries <= (gctUINT32)nodeArray[i].entries) |
| { |
| numEntries = nodeArray[i].entries; |
| |
| mStart = nodeArray[i].start; |
| mEnd = nodeArray[i].start + NumEntries - 1; |
| } |
| } |
| |
| if (mStart == -1 && mEnd == -1) |
| { |
| gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY); |
| } |
| |
| *MtlbStart = (gctUINT32)mStart; |
| *MtlbEnd = (gctUINT32)mEnd; |
| |
| OnError: |
| if (nodeArray) |
| { |
| gckOS_Free(Mmu->os, (gctPOINTER)nodeArray); |
| } |
| |
| return status; |
| } |
| |
| static gcePOOL _GetPageTablePool(IN gckOS Os) |
| { |
| gcePOOL pool = gcvPOOL_DEFAULT; |
| gctUINT64 data = 0; |
| gceSTATUS status; |
| |
| status = gckOS_QueryOption(Os, "mmuPageTablePool", &data); |
| |
| if (status == gcvSTATUS_OK) |
| { |
| if (data == 1) |
| { |
| pool = gcvPOOL_VIRTUAL; |
| } |
| } |
| |
| return pool; |
| } |
| |
| static gceSTATUS |
| _FillFlatMapping( |
| IN gckMMU Mmu, |
| IN gctUINT64 PhysBase, |
| IN gctSIZE_T Size, |
| IN gctBOOL reserved, |
| IN gctBOOL ableToShift, |
| OUT gctUINT32 *GpuBaseAddress |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 mtlb; |
| gctUINT32 physBase; |
| gckKERNEL kernel = Mmu->hardware->kernel; |
| gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K; |
| gctBOOL physicalRangeOverlapped = gcvFALSE; |
| gctBOOL virtualRangeOverlapped = gcvFALSE; |
| gctBOOL specificFlatMapping = gcvFALSE; |
| gctBOOL needShiftMapping = gcvFALSE; |
| gctUINT64 flatBase = PhysBase; |
| gctUINT32 flatSize = (gctUINT32) Size; |
| gctUINT64 base = flatBase; |
| gctUINT64 end = base + flatSize; |
| gctUINT32 reqVirtualBase = 0; |
| gctUINT32 flatVirtualBase = 0; |
| gctUINT32 i; |
| |
| /************************ Get flat mapping type and range. ************************/ |
| { |
| for (i = 0; i < Mmu->gpuPhysicalRangeCount; i++) |
| { |
| if (base < Mmu->gpuPhysicalRanges[i].start) |
| { |
| if (end > Mmu->gpuPhysicalRanges[i].start) |
| { |
| physicalRangeOverlapped = gcvTRUE; |
| if (Mmu->gpuPhysicalRanges[i].flag == gcvFLATMAP_DIRECT) |
| { |
| /* Overlapped part is direct mapping, continue direct mapping */ |
| end = Mmu->gpuPhysicalRanges[i].start; |
| } |
| else |
| { |
| /* Overlapped part is shift mapping, do entire shift mapping */ |
| needShiftMapping = gcvTRUE; |
| } |
| } |
| |
| flatSize = (gctUINT32) (end - base); |
| } |
| else if (end > Mmu->gpuPhysicalRanges[i].end) |
| { |
| if (base < Mmu->gpuPhysicalRanges[i].end) |
| { |
| physicalRangeOverlapped = gcvTRUE; |
| if (Mmu->gpuPhysicalRanges[i].flag == gcvFLATMAP_DIRECT) |
| { |
| /* Overlapped part is direct mapping, continue direct mapping */ |
| base = Mmu->gpuPhysicalRanges[i].end + 1; |
| } |
| else |
| { |
| /* Overlapped part is shift mapping, do entire shift mapping */ |
| needShiftMapping = gcvTRUE; |
| } |
| |
| } |
| |
| flatBase = base; |
| flatSize = (gctUINT32) (end - base); |
| } |
| else |
| { |
| /* it is already inside existing flat mapping ranges. */ |
| flatSize = 0; |
| } |
| |
| if (flatSize == 0) |
| { |
| if (GpuBaseAddress) |
| { |
| *GpuBaseAddress = (gctUINT32) PhysBase; |
| } |
| |
| return gcvSTATUS_OK; |
| } |
| } |
| } |
| |
| /* overwrite the orignal parameters */ |
| PhysBase = flatBase; |
| physBase = (gctUINT32)flatBase; |
| |
| mtlb = _MtlbOffset(physBase); |
| |
| if (GpuBaseAddress) |
| { |
| reqVirtualBase = *GpuBaseAddress; |
| } |
| |
| /* |
| * if no partcial physical range overlap to request entire shift mapping, |
| * it is specific shift mapping or directly mapping by default. |
| */ |
| if (!needShiftMapping) |
| { |
| flatVirtualBase = reqVirtualBase ? reqVirtualBase : (gctUINT32)flatBase; |
| } |
| |
| for (i = 0; i < Mmu->gpuAddressRangeCount; i++) |
| { |
| if (_IsRangeInsected(flatVirtualBase, flatSize, |
| Mmu->gpuAddressRanges[i].start, Mmu->gpuAddressRanges[i].size)) |
| { |
| virtualRangeOverlapped = gcvTRUE; |
| } |
| } |
| |
| /* If gpu virtual range overlapped or gpu physical over 4G, still need entire shift mapping */ |
| if ((!physicalRangeOverlapped && virtualRangeOverlapped) || |
| PhysBase + flatSize - 1 > 0xffffffff) |
| { |
| needShiftMapping = gcvTRUE; |
| } |
| |
| if (needShiftMapping && !ableToShift) |
| { |
| /* |
| * Return without mapping any address. |
| * By now, only physBase physSize could run here. |
| */ |
| return gcvSTATUS_OK; |
| } |
| |
| specificFlatMapping = (reqVirtualBase && !virtualRangeOverlapped && !physicalRangeOverlapped); |
| |
| /************************ Setup flat mapping in dynamic range. ****************/ |
| if (area->mappingStart != gcvINVALID_ADDRESS && mtlb >= area->mappingStart && mtlb < area->mappingEnd) |
| { |
| /* This path is useless now, keep it 4K page size */ |
| |
| gctUINT32_PTR stlbEntry; |
| |
| stlbEntry = _StlbEntry(area, physBase); |
| |
| /* Must be aligned to page. */ |
| gcmkASSERT((flatSize & 0xFFF) == 0); |
| |
| for (i = 0; i < (flatSize / gcdMMU_PAGE_4K_SIZE); i++) |
| { |
| /* Flat mapping in page table. */ |
| _WritePageEntry(stlbEntry, _SetPage(physBase + i * gcdMMU_PAGE_4K_SIZE, 0, gcvTRUE)); |
| #if gcdMMU_TABLE_DUMP |
| gckOS_Print("%s(%d): insert MTLB[%d] STLB[%d]: %08x\n", |
| __FUNCTION__, __LINE__, |
| (physBase & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT, |
| ((physBase & gcdMMU_STLB_4K_MASK) >> gcdMMU_STLB_4K_SHIFT) + i, |
| _ReadPageEntry(stlbEntry)); |
| #endif |
| stlbEntry++; |
| } |
| |
| #if gcdDUMP_IN_KERNEL |
| { |
| gctPHYS_ADDR_T physical; |
| gctUINT32 data = _SetPage(physBase, 0, gcvTRUE) & ~0xF; |
| gctUINT32 step = (_SetPage(physBase + gcdMMU_PAGE_4K_SIZE, 0, gcvTRUE) & ~0xF) - data; |
| gctUINT32 mask = _SetPage(physBase, 0, gcvTRUE) & 0xF; |
| |
| physical = area->stlbPhysical + 4 * _AddressToIndex(area, physBase); |
| |
| gcmkDUMP(Mmu->os, |
| "#[mmu-stlb: flat-mapping in dynamic: 0x%08X - 0x%08X]", |
| physBase, physBase - 1 + flatSize); |
| |
| gcmkDUMP(Mmu->os, |
| "@[physical.step 0x%010llX 0x%08X 0x%08lX 0x%08X 0x%08X", |
| (unsigned long long)physical, data, |
| (unsigned long)(flatSize / gcdMMU_PAGE_4K_SIZE * sizeof(gctUINT32)), |
| step, mask); |
| } |
| #endif |
| |
| /* Flat mapping in map. */ |
| _FillFlatMappingInMap(area, _AddressToIndex(area, physBase), flatSize / gcdMMU_PAGE_4K_SIZE); |
| |
| return gcvSTATUS_OK; |
| } |
| |
| /************************ Setup flat mapping in non dynamic range. **************/ |
| { |
| gctBOOL mutex = gcvFALSE; |
| gctUINT32 physBaseExt = (gctUINT32) (PhysBase >> 32); |
| gctUINT32 start = physBase & ~gcdMMU_PAGE_1M_MASK; |
| gctUINT32 end = (gctUINT32) (physBase + flatSize - 1) & ~gcdMMU_PAGE_1M_MASK; |
| gctUINT32 mStart = start >> gcdMMU_MTLB_SHIFT; |
| gctUINT32 mEnd = end >> gcdMMU_MTLB_SHIFT; |
| gctUINT32 sStart = (start & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; |
| gctUINT32 sEnd = (end & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; |
| gctPHYS_ADDR_T physical; |
| gcsMMU_STLB_CHUNK_PTR newStlbChunk = gcvNULL; |
| gctUINT32 stlbIndex = 0; |
| gctUINT32 totalNewStlbs = 0; |
| gctINT32 firstMtlbEntry = -1; |
| gctUINT32 mtlbCurEntry; |
| gcsMMU_STLB_CHUNK_PTR curStlbChunk = gcvNULL; |
| gceFLATMAP_FLAG mapFlag = gcvFLATMAP_DIRECT; |
| enum |
| { |
| COLOR_NONE = 0, |
| COLOR_RED = 1, /* occupied entry */ |
| COLOR_BLUE = 2, /* empty entry */ |
| COLOR_MAX = COLOR_BLUE, |
| } lastColor = COLOR_NONE; |
| gctUINT32 colorNumber = 0; |
| |
| /* Grab the mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); |
| mutex = gcvTRUE; |
| |
| if (needShiftMapping) |
| { |
| gctUINT32 mEntries; |
| gctUINT32 sEntries; |
| |
| mEntries = (flatSize + (1 << gcdMMU_MTLB_SHIFT) - 1) / (1 << gcdMMU_MTLB_SHIFT); |
| |
| gcmkONERROR(_GetMtlbFreeSpace(Mmu, mEntries, &mStart, &mEnd)); |
| |
| sStart = 0; |
| sEntries = (flatSize + gcdMMU_PAGE_1M_SIZE - 1) / gcdMMU_PAGE_1M_SIZE; |
| sEnd = (sEntries - 1) % gcdMMU_STLB_1M_ENTRY_NUM; |
| mapFlag = gcvFLATMAP_SHIFT; |
| } |
| |
| if (specificFlatMapping) |
| { |
| start = reqVirtualBase & ~gcdMMU_PAGE_1M_MASK; |
| end = (reqVirtualBase + flatSize - 1) & ~gcdMMU_PAGE_1M_MASK; |
| mStart = start >> gcdMMU_MTLB_SHIFT; |
| mEnd = end >> gcdMMU_MTLB_SHIFT; |
| sStart = (start & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; |
| sEnd = (end & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; |
| mapFlag = gcvFLATMAP_SHIFT; |
| } |
| |
| /* No matter direct mapping or shift mapping or specific mapping, store gpu virtual ranges */ |
| flatVirtualBase = (mStart << gcdMMU_MTLB_SHIFT) |
| | (sStart << gcdMMU_STLB_1M_SHIFT) |
| | (physBase & gcdMMU_PAGE_1M_MASK); |
| |
| /* Return GPU virtual base address if necessary */ |
| if (GpuBaseAddress) |
| { |
| *GpuBaseAddress = flatVirtualBase; |
| } |
| |
| mtlbCurEntry = mStart; |
| |
| /* find all new stlbs, part of new flat mapping range may already have stlbs*/ |
| while (mtlbCurEntry <= mEnd) |
| { |
| if (*(Mmu->mtlbLogical + mtlbCurEntry) == 0) |
| { |
| if (lastColor != COLOR_BLUE) |
| { |
| if (colorNumber < COLOR_MAX) |
| { |
| lastColor = COLOR_BLUE; |
| colorNumber++; |
| } |
| else |
| { |
| gcmkPRINT("There is a hole in new flat mapping range, which is not correct"); |
| } |
| } |
| |
| totalNewStlbs++; |
| if (-1 == firstMtlbEntry) |
| { |
| firstMtlbEntry = mtlbCurEntry; |
| } |
| } |
| else |
| { |
| if (lastColor != COLOR_RED) |
| { |
| if (colorNumber < COLOR_MAX) |
| { |
| lastColor = COLOR_RED; |
| colorNumber++; |
| } |
| else |
| { |
| gcmkPRINT("There is a hole in new flat mapping range, which is not correct"); |
| } |
| } |
| } |
| mtlbCurEntry++; |
| } |
| |
| /* Need allocate a new chunk of stlbs */ |
| if (totalNewStlbs) |
| { |
| gcePOOL pool = Mmu->pool; |
| gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS; |
| |
| gcmkONERROR( |
| gckOS_Allocate(Mmu->os, |
| sizeof(struct _gcsMMU_STLB_CHUNK), |
| (gctPOINTER *)&newStlbChunk)); |
| |
| newStlbChunk->mtlbEntryNum = totalNewStlbs; |
| newStlbChunk->next = gcvNULL; |
| newStlbChunk->videoMem = gcvNULL; |
| newStlbChunk->logical = gcvNULL; |
| newStlbChunk->size = gcdMMU_STLB_1M_SIZE * newStlbChunk->mtlbEntryNum; |
| newStlbChunk->pageCount = 0; |
| newStlbChunk->mtlbIndex = firstMtlbEntry; |
| |
| #if gcdENABLE_CACHEABLE_COMMAND_BUFFER |
| allocFlag |= gcvALLOC_FLAG_CACHEABLE; |
| #endif |
| |
| allocFlag |= gcvALLOC_FLAG_4GB_ADDR; |
| |
| gcmkONERROR(gckKERNEL_AllocateVideoMemory( |
| kernel, |
| gcdMMU_STLB_1M_SIZE, |
| gcvVIDMEM_TYPE_COMMAND, |
| allocFlag | gcvALLOC_FLAG_4K_PAGES, |
| &newStlbChunk->size, |
| &pool, |
| &newStlbChunk->videoMem)); |
| |
| /* Lock for kernel side CPU access. */ |
| gcmkONERROR(gckVIDMEM_NODE_LockCPU( |
| kernel, |
| newStlbChunk->videoMem, |
| gcvFALSE, |
| gcvFALSE, |
| (gctPOINTER *)&newStlbChunk->logical)); |
| |
| gcmkONERROR(gckOS_ZeroMemory(newStlbChunk->logical, newStlbChunk->size)); |
| |
| /* Get CPU physical address. */ |
| gcmkONERROR(gckVIDMEM_NODE_GetPhysical( |
| kernel, |
| newStlbChunk->videoMem, |
| 0, |
| &physical)); |
| |
| gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( |
| Mmu->os, |
| physical, |
| &physical)); |
| |
| newStlbChunk->physBase = physical; |
| } |
| |
| while (mStart <= mEnd) |
| { |
| gctUINT32 last = (mStart == mEnd) ? sEnd : (gcdMMU_STLB_1M_ENTRY_NUM - 1); |
| gctPHYS_ADDR_T stlbPhyBase; |
| gctUINT32_PTR stlbLogical; |
| |
| gcmkASSERT(mStart < gcdMMU_MTLB_ENTRY_NUM); |
| |
| if (*(Mmu->mtlbLogical + mStart) == 0) |
| { |
| gctUINT32 mtlbEntry; |
| curStlbChunk = newStlbChunk; |
| stlbPhyBase = curStlbChunk->physBase + (stlbIndex * gcdMMU_STLB_1M_SIZE); |
| stlbLogical = (gctUINT32_PTR)((gctUINT8_PTR)curStlbChunk->logical + (stlbIndex * gcdMMU_STLB_1M_SIZE)); |
| |
| physical = stlbPhyBase |
| /* 1MB page size */ |
| | (1 << 3) |
| /* Ignore exception */ |
| | (0 << 1) |
| /* Present */ |
| | (1 << 0); |
| |
| gcmkSAFECASTPHYSADDRT(mtlbEntry, physical); |
| |
| _WritePageEntry(Mmu->mtlbLogical + mStart, mtlbEntry); |
| |
| #if gcdMMU_TABLE_DUMP |
| gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n", |
| __FUNCTION__, __LINE__, |
| mStart, |
| _ReadPageEntry(Mmu->mtlbLogical + mStart)); |
| |
| gckOS_Print("%s(%d): STLB: logical:%08x -> physical:%08x\n", |
| __FUNCTION__, __LINE__, |
| stlbLogical, |
| stlbPhyBase); |
| #endif |
| |
| gcmkDUMP(Mmu->os, "#[mmu-mtlb: flat-mapping, slot: %d]", mStart); |
| |
| gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08X]", |
| (unsigned long long)Mmu->mtlbPhysical + mStart * 4, |
| Mmu->mtlbLogical[mStart], 4); |
| |
| ++stlbIndex; |
| } |
| else |
| { |
| gctUINT32 mtlbEntry = _ReadPageEntry(Mmu->mtlbLogical + mStart); |
| gctUINT stlbOffset; |
| |
| curStlbChunk = (gcsMMU_STLB_CHUNK_PTR)Mmu->staticSTLB; |
| |
| while (curStlbChunk) |
| { |
| if ((mStart >= curStlbChunk->mtlbIndex) && |
| (mStart < (curStlbChunk->mtlbIndex + curStlbChunk->mtlbEntryNum))) |
| { |
| break; |
| } |
| curStlbChunk = curStlbChunk->next; |
| } |
| gcmkASSERT(curStlbChunk); |
| |
| if (!curStlbChunk) |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| |
| stlbOffset = mStart - curStlbChunk->mtlbIndex; |
| |
| stlbPhyBase = curStlbChunk->physBase + (stlbOffset * gcdMMU_STLB_1M_SIZE); |
| stlbLogical = (gctUINT32_PTR)((gctUINT8_PTR)curStlbChunk->logical + (stlbOffset * gcdMMU_STLB_1M_SIZE)); |
| if (stlbPhyBase != (mtlbEntry & gcdMMU_MTLB_ENTRY_STLB_MASK)) |
| { |
| gcmkASSERT(0); |
| } |
| } |
| |
| #if gcdDUMP_IN_KERNEL |
| /* Start index. */ |
| i = sStart; |
| |
| gcmkDUMP(Mmu->os, "#[mmu-stlb: flat-mapping: 0x%08X - 0x%08X]", |
| start, start + (last - sStart) * gcdMMU_PAGE_1M_SIZE - 1); |
| #endif |
| |
| while (sStart <= last) |
| { |
| gcmkASSERT(!(start & gcdMMU_PAGE_1M_MASK)); |
| if (reserved) |
| { |
| /* program NOT_PRESENT | EXCEPTION for reserved entries */ |
| _WritePageEntry(stlbLogical + sStart, 1 << 1); |
| } |
| else |
| { |
| _WritePageEntry(stlbLogical + sStart, _SetPage(start, physBaseExt, gcvTRUE)); |
| } |
| #if gcdMMU_TABLE_DUMP |
| gckOS_Print("%s(%d): insert STLB[%d]: %08x\n", |
| __FUNCTION__, __LINE__, |
| sStart, |
| _ReadPageEntry(stlbLogical + sStart)); |
| #endif |
| /* next page. */ |
| start += gcdMMU_PAGE_1M_SIZE; |
| if (start == 0) |
| { |
| physBaseExt++; |
| } |
| sStart++; |
| curStlbChunk->pageCount++; |
| } |
| |
| #if gcdDUMP_IN_KERNEL |
| { |
| gctUINT32 data = stlbLogical[i] & ~0xF; |
| gctUINT32 step = (last > i) ? (stlbLogical[i + 1] - stlbLogical[i]) : 0; |
| gctUINT32 mask = stlbLogical[i] & 0xF; |
| |
| gcmkDUMP(Mmu->os, |
| "@[physical.step 0x%010llX 0x%08X 0x%08X 0x%08X 0x%08X]", |
| (unsigned long long)stlbPhyBase + i * 4, |
| data, (last - i) * 4, step, mask); |
| } |
| #endif |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| kernel, |
| curStlbChunk->videoMem, |
| 0, |
| curStlbChunk->logical, |
| curStlbChunk->size |
| )); |
| |
| sStart = 0; |
| ++mStart; |
| } |
| |
| gcmkASSERT(totalNewStlbs == stlbIndex); |
| |
| if (newStlbChunk) |
| { |
| /* Insert the stlbChunk into staticSTLB. */ |
| if (Mmu->staticSTLB == gcvNULL) |
| { |
| Mmu->staticSTLB = newStlbChunk; |
| } |
| else |
| { |
| gcmkASSERT(newStlbChunk != gcvNULL); |
| gcmkASSERT(newStlbChunk->next == gcvNULL); |
| newStlbChunk->next = Mmu->staticSTLB; |
| Mmu->staticSTLB = newStlbChunk; |
| } |
| } |
| |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); |
| |
| #if gcdENABLE_TRUST_APPLICATION |
| if (Mmu->hardware->options.secureMode == gcvSECURE_IN_TA) |
| { |
| gckKERNEL_SecurityMapMemory(Mmu->hardware->kernel, gcvNULL, physBase, flatSize / gcdMMU_PAGE_4K_SIZE, &physBase); |
| } |
| #endif |
| |
| /* Store the gpu physical ranges */ |
| Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].start = flatBase; |
| Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].end = flatBase + flatSize - 1; |
| Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].size = flatSize; |
| Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].flag = mapFlag; |
| Mmu->gpuPhysicalRangeCount++; |
| |
| gcmkASSERT(Mmu->gpuPhysicalRangeCount <= gcdMAX_FLAT_MAPPING_COUNT); |
| |
| /* Store the gpu virtual ranges */ |
| Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].start = flatVirtualBase; |
| Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].end = flatVirtualBase + flatSize - 1; |
| Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].size = flatSize; |
| Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].flag = mapFlag; |
| Mmu->gpuAddressRangeCount++; |
| |
| gcmkASSERT(Mmu->gpuAddressRangeCount <= gcdMAX_FLAT_MAPPING_COUNT); |
| |
| return gcvSTATUS_OK; |
| OnError: |
| /* Roll back the allocation. |
| ** We don't need roll back mtlb programming as gckmONERROR |
| ** is only used during allocation time. |
| */ |
| if (newStlbChunk) |
| { |
| if (newStlbChunk->videoMem) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( |
| kernel, |
| newStlbChunk->videoMem |
| )); |
| } |
| |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, newStlbChunk)); |
| } |
| if (mutex) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); |
| } |
| return status; |
| } |
| } |
| |
| static gceSTATUS |
| _SetupAddressArea( |
| IN gckOS Os, |
| IN gcsADDRESS_AREA_PTR Area, |
| IN gctUINT32 NumMTLBEntries |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32_PTR map; |
| gctUINT32 stlbSize = (Area->areaType == gcvAREA_TYPE_1M) |
| ? gcdMMU_STLB_1M_SIZE : gcdMMU_STLB_4K_SIZE; |
| |
| gcmkHEADER(); |
| Area->stlbSize = NumMTLBEntries * stlbSize; |
| |
| gcmkSAFECASTSIZET(Area->stlbEntries, Area->stlbSize / gcmSIZEOF(gctUINT32)); |
| |
| gcmkONERROR(gckOS_Allocate(Os, Area->stlbSize, (void **)&Area->mapLogical)); |
| |
| /* Initialization. */ |
| map = Area->mapLogical; |
| map[0] = (Area->stlbEntries << 8) | gcvMMU_FREE; |
| map[1] = ~0U; |
| Area->heapList = 0; |
| Area->freeNodes = gcvFALSE; |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _ConstructDynamicStlb( |
| IN gckMMU Mmu, |
| IN gcsADDRESS_AREA_PTR Area, |
| IN gctUINT32 NumEntries |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gctBOOL acquired = gcvFALSE; |
| gckKERNEL kernel = Mmu->hardware->kernel; |
| gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS; |
| gcePOOL pool = Mmu->pool; |
| gctUINT32 address; |
| gctUINT32 mtlbEntry; |
| gctUINT32 i; |
| |
| #if gcdENABLE_CACHEABLE_COMMAND_BUFFER |
| allocFlag |= gcvALLOC_FLAG_CACHEABLE; |
| #endif |
| |
| allocFlag |= gcvALLOC_FLAG_4GB_ADDR; |
| |
| /* Construct Slave TLB. */ |
| gcmkONERROR(gckKERNEL_AllocateVideoMemory( |
| kernel, |
| 64, |
| gcvVIDMEM_TYPE_COMMAND, |
| allocFlag | gcvALLOC_FLAG_4K_PAGES, |
| &Area->stlbSize, |
| &pool, |
| &Area->stlbVideoMem)); |
| |
| /* Lock for kernel side CPU access. */ |
| gcmkONERROR(gckVIDMEM_NODE_LockCPU( |
| kernel, |
| Area->stlbVideoMem, |
| gcvFALSE, |
| gcvFALSE, |
| (gctPOINTER *)&Area->stlbLogical)); |
| |
| #if gcdUSE_MMU_EXCEPTION |
| gcmkONERROR(_FillPageTable(Area->stlbLogical, |
| Area->stlbEntries, |
| /* Enable exception */ |
| 1 << 1)); |
| #else |
| /* Invalidate all entries. */ |
| gcmkONERROR(gckOS_ZeroMemory(Area->stlbLogical, |
| Area->stlbSize)); |
| #endif |
| |
| /* Get stlb table physical. */ |
| gcmkONERROR(gckVIDMEM_NODE_GetPhysical( |
| kernel, |
| Area->stlbVideoMem, |
| 0, |
| &Area->stlbPhysical)); |
| |
| gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Mmu->os, |
| Area->stlbPhysical, |
| &Area->stlbPhysical)); |
| |
| if (Area->areaType == gcvAREA_TYPE_1M) |
| { |
| gcmkDUMP(Mmu->os, "#[mmu: 1M page size dynamic space: 0x%08X - 0x%08X]", |
| (Area->mappingStart << gcdMMU_MTLB_SHIFT), |
| (Area->mappingEnd << gcdMMU_MTLB_SHIFT) - 1); |
| } |
| else |
| { |
| gcmkDUMP(Mmu->os, "#[mmu: 4K page size dynamic space: 0x%08X - 0x%08X]", |
| (Area->mappingStart << gcdMMU_MTLB_SHIFT), |
| (Area->mappingEnd << gcdMMU_MTLB_SHIFT) - 1); |
| } |
| |
| gcmkDUMP(Mmu->os, "#[mmu-stlb]"); |
| |
| gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08lX]", |
| (unsigned long long)Area->stlbPhysical, |
| Area->stlbLogical[0], |
| (unsigned long)Area->stlbSize); |
| |
| gcmkSAFECASTPHYSADDRT(address, Area->stlbPhysical); |
| |
| /* Grab the mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| /* Map to Master TLB. */ |
| for (i = Area->mappingStart; |
| i < Area->mappingStart + NumEntries; |
| i++) |
| { |
| if (Area->areaType == gcvAREA_TYPE_1M) |
| { |
| mtlbEntry = address |
| /* 1M page size */ |
| | (1 << 3) |
| /*Ignore exception */ |
| | (0 << 1) |
| /* Present */ |
| | (1 << 0); |
| |
| address += gcdMMU_STLB_1M_SIZE; |
| } |
| else |
| { |
| mtlbEntry = address |
| /* 4KB page size */ |
| | (0 << 2) |
| /*Ignore exception */ |
| | (0 << 1) |
| /* Present */ |
| | (1 << 0); |
| |
| address += gcdMMU_STLB_4K_SIZE; |
| } |
| |
| _WritePageEntry(Mmu->mtlbLogical + i, mtlbEntry); |
| |
| #if gcdMMU_TABLE_DUMP |
| gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n", |
| __FUNCTION__, __LINE__, |
| i, |
| _ReadPageEntry(Mmu->mtlbLogical + i)); |
| #endif |
| } |
| |
| gcmkDUMP(Mmu->os, "#[mmu-mtlb: slot: %d - %d]", |
| Area->mappingStart, Area->mappingEnd - 1); |
| |
| #if gcdDUMP_IN_KERNEL |
| { |
| gctUINT32 data = Mmu->mtlbLogical[Area->mappingStart] & ~0x3F; |
| gctUINT32 step = 0; |
| gctUINT32 mask = Mmu->mtlbLogical[Area->mappingStart] & 0x3F; |
| |
| if (NumEntries > 1) |
| { |
| step = Mmu->mtlbLogical[Area->mappingStart + 1] |
| - Mmu->mtlbLogical[Area->mappingStart]; |
| } |
| |
| gcmkDUMP(Mmu->os, |
| "@[physical.step 0x%010llX 0x%08X 0x%08X 0x%08X 0x%08X]", |
| (unsigned long long)(Mmu->mtlbPhysical + Area->mappingStart * 4), |
| data, NumEntries * 4, step, mask); |
| } |
| #endif |
| |
| OnError: |
| if (acquired) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); |
| } |
| |
| return status; |
| } |
| |
| gceSTATUS |
| gckMMU_SetupDynamicSpace( |
| IN gckMMU Mmu |
| ) |
| { |
| gceSTATUS status; |
| gcsFreeSpaceNode_PTR nodeArray = gcvNULL; |
| gctINT i, nodeArraySize = 0; |
| gctINT numEntries = 0; |
| gckKERNEL kernel = Mmu->hardware->kernel; |
| gcsADDRESS_AREA_PTR area4K = &Mmu->dynamicArea4K; |
| #if gcdENABLE_GPU_1M_PAGE |
| gcsADDRESS_AREA_PTR area1M = &Mmu->dynamicArea1M; |
| gctINT numEntries1M; |
| #endif |
| |
| /* Find all the free address space. */ |
| gcmkONERROR(_CollectFreeSpace(Mmu, &nodeArray, &nodeArraySize)); |
| |
| for (i = 0; i < nodeArraySize; i++) |
| { |
| if (nodeArray[i].entries > numEntries) |
| { |
| area4K->mappingStart = nodeArray[i].start; |
| numEntries = nodeArray[i].entries; |
| area4K->mappingEnd = area4K->mappingStart + numEntries - 1; |
| } |
| } |
| |
| gckOS_Free(Mmu->os, (gctPOINTER)nodeArray); |
| |
| #if gcdENABLE_TRUST_APPLICATION |
| if (gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_SECURITY) == gcvSTATUS_TRUE) |
| { |
| /* Setup secure address area when needed. */ |
| gctUINT32 secureAreaSize = gcdMMU_SECURE_AREA_SIZE; |
| gcsADDRESS_AREA_PTR secureArea = &Mmu->secureArea; |
| |
| gcmkASSERT(numEntries > (gctINT)secureAreaSize); |
| |
| secureArea->mappingStart = area4K->mappingStart |
| + (numEntries - secureAreaSize); |
| |
| gcmkONERROR(_SetupAddressArea(Mmu->os, secureArea, secureAreaSize)); |
| |
| numEntries -= secureAreaSize; |
| area4K->mappingEnd -= secureAreaSize; |
| } |
| #endif |
| |
| #if gcdENABLE_GPU_1M_PAGE |
| numEntries1M = numEntries >> 1; |
| area1M->mappingStart = area4K->mappingStart + (numEntries - numEntries1M); |
| area1M->mappingEnd = area1M->mappingStart + numEntries1M - 1; |
| area1M->areaType = gcvAREA_TYPE_1M; |
| numEntries -= numEntries1M; |
| area4K->mappingEnd -= numEntries1M; |
| |
| gcmkONERROR(_SetupAddressArea(Mmu->os, area1M, numEntries1M)); |
| |
| gcmkONERROR(_ConstructDynamicStlb(Mmu, area1M, numEntries1M)); |
| #endif |
| |
| area4K->areaType = gcvAREA_TYPE_4K; |
| |
| /* Setup normal address area. */ |
| gcmkONERROR(_SetupAddressArea(Mmu->os, area4K, numEntries)); |
| |
| gcmkONERROR(_ConstructDynamicStlb(Mmu, area4K, numEntries)); |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| #if gcdENABLE_GPU_1M_PAGE |
| if (area1M->mapLogical) |
| { |
| gcmkVERIFY_OK( |
| gckOS_Free(Mmu->os, (gctPOINTER) area1M->mapLogical)); |
| } |
| |
| if (area1M->stlbVideoMem) |
| { |
| gcmkVERIFY_OK( |
| gckVIDMEM_NODE_Dereference(kernel, |
| area1M->stlbVideoMem)); |
| } |
| #endif |
| |
| if (area4K->mapLogical) |
| { |
| gcmkVERIFY_OK( |
| gckOS_Free(Mmu->os, (gctPOINTER) area4K->mapLogical)); |
| } |
| |
| if (area4K->stlbVideoMem) |
| { |
| gcmkVERIFY_OK( |
| gckVIDMEM_NODE_Dereference(kernel, |
| area4K->stlbVideoMem)); |
| } |
| |
| return status; |
| } |
| |
| gctUINT32 |
| _GetPageCountOfUsedNode( |
| gctUINT32_PTR Node |
| ) |
| { |
| gctUINT32 count; |
| |
| count = gcmENTRY_COUNT(*Node); |
| |
| if ((count << 8) == (~((1U<<8)-1))) |
| { |
| count = 1; |
| } |
| |
| return count; |
| } |
| |
| static gcsADDRESS_AREA_PTR |
| _GetProcessArea( |
| IN gckMMU Mmu, |
| IN gcePAGE_TYPE PageType, |
| IN gctBOOL Secure |
| ) |
| { |
| #if gcdENABLE_TRUST_APPLICATION |
| if (Secure == gcvTRUE) |
| { |
| return &Mmu->secureArea; |
| } |
| #endif |
| |
| if (PageType == gcvPAGE_TYPE_1M) |
| { |
| return &Mmu->dynamicArea1M; |
| } |
| else |
| { |
| return &Mmu->dynamicArea4K; |
| } |
| } |
| |
| /******************************************************************************* |
| ** |
| ** _Construct |
| ** |
| ** Construct a new gckMMU object. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** gctSIZE_T MmuSize |
| ** Number of bytes for the page table. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckMMU * Mmu |
| ** Pointer to a variable that receives the gckMMU object pointer. |
| */ |
| gceSTATUS |
| _Construct( |
| IN gckKERNEL Kernel, |
| IN gctSIZE_T MmuSize, |
| OUT gckMMU * Mmu |
| ) |
| { |
| gckOS os; |
| gckHARDWARE hardware; |
| gceSTATUS status; |
| gckMMU mmu = gcvNULL; |
| gctUINT32_PTR map; |
| gctPOINTER pointer = gcvNULL; |
| gctPHYS_ADDR_T physBase; |
| gctSIZE_T physSize; |
| gctPHYS_ADDR_T contiguousBase; |
| gctSIZE_T contiguousSize = 0; |
| gctPHYS_ADDR_T externalBase; |
| gctSIZE_T externalSize = 0; |
| gctUINT32 gpuAddress; |
| gctPHYS_ADDR_T gpuPhysical; |
| gcsADDRESS_AREA_PTR area = gcvNULL; |
| gcePOOL pool; |
| gctUINT64 data; |
| gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS; |
| gctUINT64 mmuEnabled; |
| |
| gcmkHEADER_ARG("Kernel=0x%x MmuSize=%lu", Kernel, MmuSize); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); |
| gcmkVERIFY_ARGUMENT(MmuSize > 0); |
| gcmkVERIFY_ARGUMENT(Mmu != gcvNULL); |
| |
| /* Extract the gckOS object pointer. */ |
| os = Kernel->os; |
| gcmkVERIFY_OBJECT(os, gcvOBJ_OS); |
| |
| /* Extract the gckHARDWARE object pointer. */ |
| hardware = Kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Allocate memory for the gckMMU object. */ |
| gcmkONERROR(gckOS_Allocate(os, sizeof(struct _gckMMU), &pointer)); |
| |
| gckOS_ZeroMemory(pointer, sizeof(struct _gckMMU)); |
| |
| mmu = pointer; |
| |
| /* Initialize the gckMMU object. */ |
| mmu->object.type = gcvOBJ_MMU; |
| mmu->os = os; |
| mmu->hardware = hardware; |
| mmu->pageTableMutex = gcvNULL; |
| mmu->mtlbLogical = gcvNULL; |
| mmu->staticSTLB = gcvNULL; |
| mmu->enabled = gcvFALSE; |
| |
| mmu->dynamicAreaSetuped = gcvFALSE; |
| mmu->pool = _GetPageTablePool(mmu->os); |
| gcsLIST_Init(&mmu->hardwareList); |
| |
| /* Use 4K page size for MMU version 0. */ |
| area = &mmu->dynamicArea4K; |
| area->mapLogical = gcvNULL; |
| area->stlbLogical = gcvNULL; |
| |
| /* Create the page table mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &mmu->pageTableMutex)); |
| |
| gcmkONERROR(gckOS_QueryOption(os, "mmu", &mmuEnabled)); |
| |
| if (hardware->mmuVersion == 0) |
| { |
| area->stlbSize = MmuSize; |
| |
| /* Construct address space management table. */ |
| gcmkONERROR(gckOS_Allocate(mmu->os, |
| area->stlbSize, |
| &pointer)); |
| |
| area->mapLogical = pointer; |
| |
| pool = mmu->pool; |
| |
| #if gcdENABLE_CACHEABLE_COMMAND_BUFFER |
| allocFlag |= gcvALLOC_FLAG_CACHEABLE; |
| #endif |
| |
| /* Construct page table read by GPU. */ |
| gcmkONERROR(gckKERNEL_AllocateVideoMemory( |
| Kernel, |
| 4096, |
| gcvVIDMEM_TYPE_COMMAND, |
| allocFlag, |
| &area->stlbSize, |
| &pool, |
| &area->stlbVideoMem)); |
| |
| /* Lock for kernel side CPU access. */ |
| gcmkONERROR(gckVIDMEM_NODE_LockCPU( |
| Kernel, |
| area->stlbVideoMem, |
| gcvFALSE, |
| gcvFALSE, |
| &pointer)); |
| |
| area->stlbLogical = pointer; |
| |
| /* Get CPU physical address. */ |
| gcmkONERROR(gckVIDMEM_NODE_GetPhysical( |
| Kernel, |
| area->stlbVideoMem, |
| 0, |
| &area->stlbPhysical)); |
| |
| gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(mmu->os, |
| area->stlbPhysical, |
| &area->stlbPhysical)); |
| |
| /* Compute number of entries in page table. */ |
| gcmkSAFECASTSIZET(area->stlbEntries, area->stlbSize / sizeof(gctUINT32)); |
| |
| /* Mark all pages as free. */ |
| map = area->mapLogical; |
| |
| _FillPageTable(area->stlbLogical, area->stlbEntries, mmu->safeAddress); |
| |
| gcmkDUMP(mmu->os, |
| "#[mmu0: fill with safe address]"); |
| |
| gcmkDUMP(mmu->os, |
| "@[physical.fill 0x%010llX 0x%08X 0x%08lX]", |
| (unsigned long long)area->stlbPhysical, |
| area->stlbLogical[0], |
| (unsigned long)area->stlbSize); |
| |
| map[0] = (area->stlbEntries << 8) | gcvMMU_FREE; |
| map[1] = ~0U; |
| area->heapList = 0; |
| area->freeNodes = gcvFALSE; |
| |
| status = gckOS_QueryOption(mmu->os, "contiguousBase", &contiguousBase); |
| |
| if (gcmIS_SUCCESS(status)) |
| { |
| status = gckOS_QueryOption(mmu->os, "contiguousSize", &data); |
| contiguousSize = (gctSIZE_T)data; |
| } |
| |
| if (gcmIS_SUCCESS(status) && contiguousSize) |
| { |
| /* Convert to GPU address. */ |
| mmu->contiguousBaseAddress = (gctUINT32)(contiguousBase - Kernel->hardware->baseAddress); |
| } |
| } |
| else |
| { |
| mmu->mtlbSize = gcdMMU_MTLB_SIZE; |
| |
| pool = mmu->pool; |
| |
| #if gcdENABLE_CACHEABLE_COMMAND_BUFFER |
| allocFlag |= gcvALLOC_FLAG_CACHEABLE; |
| #endif |
| |
| allocFlag |= gcvALLOC_FLAG_4GB_ADDR; |
| |
| /* 1K mode is 1024 byte aligned. */ |
| gcmkONERROR(gckKERNEL_AllocateVideoMemory( |
| Kernel, |
| 1024, |
| gcvVIDMEM_TYPE_COMMAND, |
| allocFlag | gcvALLOC_FLAG_4K_PAGES, |
| &mmu->mtlbSize, |
| &pool, |
| &mmu->mtlbVideoMem)); |
| |
| /* Lock for kernel side CPU access. */ |
| gcmkONERROR(gckVIDMEM_NODE_LockCPU( |
| Kernel, |
| mmu->mtlbVideoMem, |
| gcvFALSE, |
| gcvFALSE, |
| &pointer)); |
| |
| mmu->mtlbLogical = pointer; |
| |
| mmu->dynamicArea4K.mappingStart = gcvINVALID_ADDRESS; |
| mmu->dynamicArea1M.mappingStart = gcvINVALID_ADDRESS; |
| |
| /* Get mtlb table physical. */ |
| gcmkONERROR(gckVIDMEM_NODE_GetPhysical( |
| Kernel, |
| mmu->mtlbVideoMem, |
| 0, |
| &mmu->mtlbPhysical)); |
| |
| gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( |
| mmu->os, |
| mmu->mtlbPhysical, |
| &mmu->mtlbPhysical)); |
| |
| /* Invalid all the entries. */ |
| gcmkONERROR( |
| gckOS_ZeroMemory(pointer, mmu->mtlbSize)); |
| |
| gcmkONERROR( |
| gckOS_QueryOption(mmu->os, "physBase", &physBase)); |
| |
| gcmkONERROR( |
| gckOS_QueryOption(mmu->os, "physSize", &data)); |
| physSize = (gctSIZE_T)data; |
| |
| gcmkONERROR( |
| gckOS_CPUPhysicalToGPUPhysical(mmu->os, physBase, &gpuPhysical)); |
| |
| gcmkSAFECASTPHYSADDRT(gpuAddress, gpuPhysical); |
| |
| if (physSize) |
| { |
| /* Setup user specified flat mapping. */ |
| gcmkONERROR(_FillFlatMapping(mmu, gpuAddress, physSize, gcvFALSE, gcvFALSE, gcvNULL)); |
| } |
| |
| #if !(0 || gcdCAPTURE_ONLY_MODE) |
| if (!_ReadPageEntry(mmu->mtlbLogical + 0)) |
| { |
| gctUINT32 mtlbEntry; |
| /* |
| * Reserved the first mtlb. |
| * 1MB page size, Ingore exception, Not Present. |
| */ |
| mtlbEntry = (1 << 3) |
| | (0 << 1) |
| | (0 << 0); |
| |
| _WritePageEntry(mmu->mtlbLogical + 0, mtlbEntry); |
| |
| gcmkDUMP(mmu->os, "#[mmu-mtlb: reserved 16M space, slot: 0]"); |
| gcmkDUMP(mmu->os, |
| "@[physical.fill 0x%010llX 0x%08X 0x%08X]", |
| (unsigned long long)mmu->mtlbPhysical, |
| mmu->mtlbLogical[0], 4); |
| |
| /* Store the gpu virtual ranges */ |
| mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].start = 0; |
| mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].end = (16 << 20) - 1; |
| mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].size = (16 << 20); |
| mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].flag = gcvFLATMAP_DIRECT; |
| mmu->gpuAddressRangeCount++; |
| } |
| #endif |
| |
| status = gckOS_QueryOption(mmu->os, "contiguousBase", &contiguousBase); |
| |
| if (gcmIS_SUCCESS(status)) |
| { |
| status = gckOS_QueryOption(mmu->os, "contiguousSize", &data); |
| contiguousSize = (gctSIZE_T)data; |
| } |
| |
| if (gcmIS_SUCCESS(status) && contiguousSize) |
| { |
| gctUINT64 gpuContiguousBase; |
| gctUINT32 contiguousBaseAddress = 0; |
| |
| gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(mmu->os, contiguousBase, &gpuContiguousBase)); |
| |
| /* Setup flat mapping for reserved memory (VIDMEM). */ |
| gcmkONERROR(_FillFlatMapping(mmu, gpuContiguousBase, contiguousSize, gcvFALSE, gcvTRUE, &contiguousBaseAddress)); |
| |
| if (mmuEnabled) |
| { |
| mmu->contiguousBaseAddress = contiguousBaseAddress; |
| } |
| else |
| { |
| gcmkSAFECASTPHYSADDRT(mmu->contiguousBaseAddress, gpuContiguousBase); |
| } |
| } |
| |
| status = gckOS_QueryOption(mmu->os, "externalBase", &externalBase); |
| |
| if (gcmIS_SUCCESS(status)) |
| { |
| status = gckOS_QueryOption(mmu->os, "externalSize", &data); |
| externalSize = (gctSIZE_T)data; |
| } |
| |
| if (gcmIS_SUCCESS(status) && externalSize) |
| { |
| gctUINT64 gpuExternalBase; |
| gctUINT32 externalBaseAddress = 0; |
| |
| gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(mmu->os, externalBase, &gpuExternalBase)); |
| |
| /* Setup flat mapping for external memory. */ |
| gcmkONERROR(_FillFlatMapping(mmu, gpuExternalBase, externalSize, gcvFALSE, gcvTRUE, &externalBaseAddress)); |
| |
| mmu->externalBaseAddress = externalBaseAddress; |
| } |
| } |
| |
| /* A 64 byte for safe address, we use 256 here. */ |
| mmu->safePageSize = 256; |
| |
| pool = mmu->pool; |
| |
| /* Allocate safe page from video memory. */ |
| gcmkONERROR(gckKERNEL_AllocateVideoMemory( |
| Kernel, |
| 256, |
| gcvVIDMEM_TYPE_COMMAND, |
| gcvALLOC_FLAG_CONTIGUOUS | gcvALLOC_FLAG_4K_PAGES | gcvALLOC_FLAG_4GB_ADDR, |
| &mmu->safePageSize, |
| &pool, |
| &mmu->safePageVideoMem |
| )); |
| |
| /* Lock for kernel side CPU access. */ |
| gcmkONERROR(gckVIDMEM_NODE_LockCPU( |
| Kernel, |
| mmu->safePageVideoMem, |
| gcvFALSE, |
| gcvFALSE, |
| &mmu->safePageLogical |
| )); |
| |
| /* Get CPU physical address. */ |
| gcmkONERROR(gckVIDMEM_NODE_GetPhysical( |
| Kernel, |
| mmu->safePageVideoMem, |
| 0, |
| &mmu->safePagePhysical |
| )); |
| |
| gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( |
| os, |
| mmu->safePagePhysical, |
| &mmu->safePagePhysical |
| )); |
| |
| gcmkSAFECASTPHYSADDRT(mmu->safeAddress, mmu->safePagePhysical); |
| |
| gckOS_ZeroMemory(mmu->safePageLogical, mmu->safePageSize); |
| |
| gcmkDUMP(mmu->os, "#[safe page]"); |
| gcmkDUMP(mmu->os, |
| "@[physical.fill 0x%010llX 0x%08X 0x%08lX]", |
| (unsigned long long)mmu->safePagePhysical, |
| 0, |
| (unsigned long)mmu->safePageSize); |
| |
| gcmkDUMP_BUFFER( |
| mmu->os, |
| gcvDUMP_BUFFER_KERNEL_COMMAND, |
| mmu->safePageLogical, |
| mmu->safeAddress, |
| mmu->safePageSize |
| ); |
| |
| gcmkONERROR(gckQUEUE_Allocate(os, &mmu->recentFreedAddresses, 16)); |
| |
| mmu->sRAMMapped = gcvFALSE; |
| |
| /* Return the gckMMU object pointer. */ |
| *Mmu = mmu; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Mmu=0x%x", *Mmu); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Roll back. */ |
| if (mmu != gcvNULL) |
| { |
| if (area != gcvNULL && area->mapLogical != gcvNULL) |
| { |
| gcmkVERIFY_OK( |
| gckOS_Free(os, (gctPOINTER) area->mapLogical)); |
| |
| gcmkVERIFY_OK( |
| gckVIDMEM_NODE_Dereference(Kernel, |
| area->stlbVideoMem)); |
| } |
| |
| if (mmu->mtlbLogical != gcvNULL) |
| { |
| gcmkVERIFY_OK( |
| gckVIDMEM_NODE_Dereference(Kernel, |
| mmu->mtlbVideoMem)); |
| } |
| |
| if (mmu->pageTableMutex != gcvNULL) |
| { |
| /* Delete the mutex. */ |
| gcmkVERIFY_OK( |
| gckOS_DeleteMutex(os, mmu->pageTableMutex)); |
| } |
| |
| gcmkVERIFY_OK(gckQUEUE_Free(os, &mmu->recentFreedAddresses)); |
| |
| /* Mark the gckMMU object as unknown. */ |
| mmu->object.type = gcvOBJ_UNKNOWN; |
| |
| /* Free the allocates memory. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, mmu)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _FreeAddressArea( |
| gckKERNEL Kernel, |
| gcsADDRESS_AREA * Area |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| |
| if (Area->mapLogical != gcvNULL) |
| { |
| gcmkVERIFY_OK( |
| gckOS_Free(Kernel->os, (gctPOINTER) Area->mapLogical)); |
| } |
| |
| if (Area->stlbLogical != gcvNULL) |
| { |
| /* Free page table. */ |
| gcmkVERIFY_OK( |
| gckVIDMEM_NODE_Dereference(Kernel, |
| Area->stlbVideoMem)); |
| } |
| |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** _Destroy |
| ** |
| ** Destroy a gckMMU object. |
| ** |
| ** INPUT: |
| ** |
| ** gckMMU Mmu |
| ** Pointer to an gckMMU object. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| _Destroy( |
| IN gckMMU Mmu |
| ) |
| { |
| gckKERNEL kernel = Mmu->hardware->kernel; |
| |
| gcmkHEADER_ARG("Mmu=0x%x", Mmu); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); |
| |
| while (Mmu->staticSTLB != gcvNULL) |
| { |
| gcsMMU_STLB_CHUNK_PTR pre = Mmu->staticSTLB; |
| Mmu->staticSTLB = pre->next; |
| |
| if (pre->videoMem) |
| { |
| gcmkVERIFY_OK( |
| gckVIDMEM_NODE_Dereference(kernel, |
| pre->videoMem)); |
| } |
| |
| if (pre->mtlbEntryNum != 0) |
| { |
| gctUINT i; |
| for (i = 0; i < pre->mtlbEntryNum; ++i) |
| { |
| _WritePageEntry(Mmu->mtlbLogical + pre->mtlbIndex + i, 0); |
| #if gcdMMU_TABLE_DUMP |
| gckOS_Print("%s(%d): clean MTLB[%d]\n", |
| __FUNCTION__, __LINE__, |
| pre->mtlbIndex + i); |
| #endif |
| } |
| |
| gcmkDUMP(Mmu->os, |
| "#[mmu-mtlb: clean up slot: %d - %d]", |
| pre->mtlbIndex, |
| pre->mtlbIndex + pre->mtlbEntryNum - 1); |
| |
| gcmkDUMP(Mmu->os, |
| "@[physical.fill 0x%010llX 0x%08X 0x%08lX]", |
| (unsigned long long)(Mmu->mtlbPhysical + pre->mtlbIndex * 4), |
| Mmu->mtlbLogical[pre->mtlbIndex], |
| (unsigned long)(pre->mtlbEntryNum * 4)); |
| } |
| |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, pre)); |
| } |
| |
| if (Mmu->hardware->mmuVersion != 0) |
| { |
| gcmkVERIFY_OK( |
| gckVIDMEM_NODE_Dereference(kernel, |
| Mmu->mtlbVideoMem)); |
| } |
| |
| /* Free address area. */ |
| gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->dynamicArea4K)); |
| #if gcdENABLE_GPU_1M_PAGE |
| gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->dynamicArea1M)); |
| #endif |
| gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->secureArea)); |
| |
| /* Delete the page table mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Mmu->os, Mmu->pageTableMutex)); |
| |
| if (Mmu->safePageLogical != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( |
| kernel, |
| Mmu->safePageVideoMem |
| )); |
| } |
| |
| gcmkVERIFY_OK(gckQUEUE_Free(Mmu->os, &Mmu->recentFreedAddresses)); |
| |
| /* Mark the gckMMU object as unknown. */ |
| Mmu->object.type = gcvOBJ_UNKNOWN; |
| |
| /* Free the gckMMU object. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, Mmu)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /******************************************************************************* |
| ** _AdjstIndex |
| ** |
| ** Adjust the index from which we search for a usable node to make sure |
| ** index allocated is greater than Start. |
| */ |
| gceSTATUS |
| _AdjustIndex( |
| IN gckMMU Mmu, |
| IN gctUINT32 Index, |
| IN gctUINT32 PageCount, |
| IN gctUINT32 Start, |
| OUT gctUINT32 * IndexAdjusted |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 index = Index; |
| gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K; |
| gctUINT32_PTR map = area->mapLogical; |
| |
| gcmkHEADER(); |
| |
| for (; index < area->stlbEntries;) |
| { |
| gctUINT32 result = 0; |
| gctUINT32 nodeSize = 0; |
| |
| if (index >= Start) |
| { |
| break; |
| } |
| |
| switch (gcmENTRY_TYPE(map[index])) |
| { |
| case gcvMMU_SINGLE: |
| nodeSize = 1; |
| break; |
| |
| case gcvMMU_FREE: |
| nodeSize = map[index] >> 8; |
| break; |
| |
| default: |
| gcmkFATAL("MMU table correcupted at index %u!", index); |
| gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); |
| } |
| |
| if (nodeSize > PageCount) |
| { |
| result = index + (nodeSize - PageCount); |
| |
| if (result >= Start) |
| { |
| break; |
| } |
| } |
| |
| switch (gcmENTRY_TYPE(map[index])) |
| { |
| case gcvMMU_SINGLE: |
| index = map[index] >> 8; |
| break; |
| |
| case gcvMMU_FREE: |
| index = map[index + 1]; |
| break; |
| |
| default: |
| gcmkFATAL("MMU table correcupted at index %u!", index); |
| gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); |
| } |
| } |
| |
| *IndexAdjusted = index; |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckMMU_Construct( |
| IN gckKERNEL Kernel, |
| IN gctSIZE_T MmuSize, |
| OUT gckMMU * Mmu |
| ) |
| { |
| #if gcdSHARED_PAGETABLE |
| gceSTATUS status; |
| gctPOINTER pointer; |
| |
| gcmkHEADER_ARG("Kernel=0x%08x", Kernel); |
| |
| if (sharedPageTable == gcvNULL) |
| { |
| gcmkONERROR( |
| gckOS_Allocate(Kernel->os, |
| sizeof(struct _gcsSharedPageTable), |
| &pointer)); |
| sharedPageTable = pointer; |
| |
| gcmkONERROR( |
| gckOS_ZeroMemory(sharedPageTable, |
| sizeof(struct _gcsSharedPageTable))); |
| |
| gcmkONERROR(_Construct(Kernel, MmuSize, &sharedPageTable->mmu)); |
| } |
| |
| *Mmu = sharedPageTable->mmu; |
| |
| sharedPageTable->hardwares[sharedPageTable->reference] = Kernel->hardware; |
| |
| sharedPageTable->reference++; |
| |
| gcmkFOOTER_ARG("sharedPageTable->reference=%lu", sharedPageTable->reference); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (sharedPageTable) |
| { |
| if (sharedPageTable->mmu) |
| { |
| gcmkVERIFY_OK(gckMMU_Destroy(sharedPageTable->mmu)); |
| } |
| |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, sharedPageTable)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| #else |
| return _Construct(Kernel, MmuSize, Mmu); |
| #endif |
| } |
| |
| gceSTATUS |
| gckMMU_Destroy( |
| IN gckMMU Mmu |
| ) |
| { |
| #if gcdSHARED_PAGETABLE |
| gckOS os = Mmu->os; |
| |
| sharedPageTable->reference--; |
| |
| if (sharedPageTable->reference == 0) |
| { |
| if (sharedPageTable->mmu) |
| { |
| gcmkVERIFY_OK(_Destroy(Mmu)); |
| } |
| |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, sharedPageTable)); |
| } |
| |
| return gcvSTATUS_OK; |
| #else |
| return _Destroy(Mmu); |
| #endif |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckMMU_AllocatePages |
| ** |
| ** Allocate pages inside the page table. |
| ** |
| ** INPUT: |
| ** |
| ** gckMMU Mmu |
| ** Pointer to an gckMMU object. |
| ** |
| ** gctSIZE_T PageCount |
| ** Number of pages to allocate. |
| ** |
| ** OUTPUT: |
| ** |
| ** gctPOINTER * PageTable |
| ** Pointer to a variable that receives the base address of the page |
| ** table. |
| ** |
| ** gctUINT32 * Address |
| ** Pointer to a variable that receives the hardware specific address. |
| */ |
| gceSTATUS |
| _AllocatePages( |
| IN gckMMU Mmu, |
| IN gctSIZE_T PageCount, |
| IN gceVIDMEM_TYPE Type, |
| IN gcePAGE_TYPE PageType, |
| IN gctBOOL Secure, |
| OUT gctPOINTER * PageTable, |
| OUT gctUINT32 * Address |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL mutex = gcvFALSE; |
| gctUINT32 index = 0, previous = ~0U, left; |
| gctUINT32_PTR map; |
| gctBOOL gotIt; |
| gctUINT32 address; |
| gctUINT32 pageCount; |
| gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, Secure); |
| |
| gcmkHEADER_ARG("Mmu=0x%x PageCount=%lu", Mmu, PageCount); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); |
| gcmkVERIFY_ARGUMENT(PageCount > 0); |
| gcmkVERIFY_ARGUMENT(PageTable != gcvNULL); |
| |
| if (PageCount > area->stlbEntries) |
| { |
| /* Not enough pages avaiable. */ |
| gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); |
| } |
| |
| gcmkSAFECASTSIZET(pageCount, PageCount); |
| |
| #if gcdBOUNDARY_CHECK |
| /* Extra pages as bounary. */ |
| pageCount += gcdBOUNDARY_CHECK * 2; |
| #endif |
| |
| /* Grab the mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); |
| mutex = gcvTRUE; |
| |
| /* Cast pointer to page table. */ |
| for (map = area->mapLogical, gotIt = gcvFALSE; !gotIt;) |
| { |
| index = area->heapList; |
| |
| if ((Mmu->hardware->mmuVersion == 0) && |
| (Type == gcvVIDMEM_TYPE_VERTEX_BUFFER)) |
| { |
| gcmkONERROR(_AdjustIndex( |
| Mmu, |
| index, |
| pageCount, |
| gcdVERTEX_START / gcmSIZEOF(gctUINT32), |
| &index |
| )); |
| } |
| |
| /* Walk the heap list. */ |
| for (; !gotIt && (index < area->stlbEntries);) |
| { |
| /* Check the node type. */ |
| switch (gcmENTRY_TYPE(map[index])) |
| { |
| case gcvMMU_SINGLE: |
| /* Single odes are valid if we only need 1 page. */ |
| if (pageCount == 1) |
| { |
| gotIt = gcvTRUE; |
| } |
| else |
| { |
| /* Move to next node. */ |
| previous = index; |
| index = map[index] >> 8; |
| } |
| break; |
| |
| case gcvMMU_FREE: |
| /* Test if the node has enough space. */ |
| if (pageCount <= (map[index] >> 8)) |
| { |
| gotIt = gcvTRUE; |
| } |
| else |
| { |
| /* Move to next node. */ |
| previous = index; |
| index = map[index + 1]; |
| } |
| break; |
| |
| default: |
| gcmkFATAL("MMU table correcupted at index %u!", index); |
| gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); |
| } |
| } |
| |
| /* Test if we are out of memory. */ |
| if (index >= area->stlbEntries) |
| { |
| if (area->freeNodes) |
| { |
| /* Time to move out the trash! */ |
| gcmkONERROR(_Collect(area)); |
| |
| /* We are going to search from start, so reset previous to start. */ |
| previous = ~0U; |
| } |
| else |
| { |
| /* Out of resources. */ |
| gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); |
| } |
| } |
| } |
| |
| switch (gcmENTRY_TYPE(map[index])) |
| { |
| case gcvMMU_SINGLE: |
| /* Unlink single node from free list. */ |
| gcmkONERROR( |
| _Link(area, previous, map[index] >> 8)); |
| break; |
| |
| case gcvMMU_FREE: |
| /* Check how many pages will be left. */ |
| left = (map[index] >> 8) - pageCount; |
| switch (left) |
| { |
| case 0: |
| /* The entire node is consumed, just unlink it. */ |
| gcmkONERROR( |
| _Link(area, previous, map[index + 1])); |
| break; |
| |
| case 1: |
| /* One page will remain. Convert the node to a single node and |
| ** advance the index. */ |
| map[index] = (map[index + 1] << 8) | gcvMMU_SINGLE; |
| index ++; |
| break; |
| |
| default: |
| /* Enough pages remain for a new node. However, we will just adjust |
| ** the size of the current node and advance the index. */ |
| map[index] = (left << 8) | gcvMMU_FREE; |
| index += left; |
| break; |
| } |
| break; |
| } |
| |
| /* Mark node as used. */ |
| gcmkONERROR(_FillMap(&map[index], pageCount, gcvMMU_USED)); |
| |
| #if gcdBOUNDARY_CHECK |
| index += gcdBOUNDARY_CHECK; |
| #endif |
| |
| /* Record pageCount of allocated node at the beginning of node. */ |
| if (pageCount == 1) |
| { |
| map[index] = (~((1U<<8)-1)) | gcvMMU_USED; |
| } |
| else |
| { |
| map[index] = (pageCount << 8) | gcvMMU_USED; |
| } |
| |
| if (area->stlbLogical != gcvNULL) |
| { |
| /* Return pointer to page table. */ |
| *PageTable = &area->stlbLogical[index]; |
| } |
| else |
| { |
| /* Page table for secure area is handled in trust application. */ |
| *PageTable = gcvNULL; |
| } |
| |
| /* Build virtual address. */ |
| if (Mmu->hardware->mmuVersion == 0) |
| { |
| gcmkONERROR( |
| gckHARDWARE_BuildVirtualAddress(Mmu->hardware, index, 0, &address)); |
| } |
| else |
| { |
| gctUINT32 num = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM; |
| gctUINT32 shift = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT; |
| gctUINT32 masterOffset = index / num + area->mappingStart; |
| gctUINT32 slaveOffset = index % num; |
| |
| address = (masterOffset << gcdMMU_MTLB_SHIFT) |
| | (slaveOffset << shift); |
| } |
| |
| if (Address != gcvNULL) |
| { |
| *Address = address; |
| } |
| |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*PageTable=0x%x *Address=%08x", |
| *PageTable, gcmOPT_VALUE(Address)); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| |
| if (mutex) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckMMU_FreePages |
| ** |
| ** Free pages inside the page table. |
| ** |
| ** INPUT: |
| ** |
| ** gckMMU Mmu |
| ** Pointer to an gckMMU object. |
| ** |
| ** gctPOINTER PageTable |
| ** Base address of the page table to free. |
| ** |
| ** gctSIZE_T PageCount |
| ** Number of pages to free. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| _FreePages( |
| IN gckMMU Mmu, |
| IN gctBOOL Secure, |
| IN gcePAGE_TYPE PageType, |
| IN gctUINT32 Address, |
| IN gctPOINTER PageTable, |
| IN gctSIZE_T PageCount |
| ) |
| { |
| gctUINT32 index; |
| gctUINT32_PTR node; |
| gceSTATUS status; |
| gctBOOL acquired = gcvFALSE; |
| gctUINT32 pageCount; |
| gcuQUEUEDATA data; |
| gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, gcvFALSE); |
| gctUINT32 pageSize = (PageType == gcvPAGE_TYPE_1M) |
| ? gcdMMU_PAGE_1M_SIZE : gcdMMU_PAGE_4K_SIZE; |
| |
| gcmkHEADER_ARG("Mmu=0x%x PageTable=0x%x PageCount=%lu", |
| Mmu, PageTable, PageCount); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); |
| gcmkVERIFY_ARGUMENT(PageCount > 0); |
| |
| gcmkSAFECASTSIZET(pageCount, PageCount); |
| |
| #if gcdBOUNDARY_CHECK |
| pageCount += gcdBOUNDARY_CHECK * 2; |
| #endif |
| |
| /* Get the node by index. */ |
| index = (gctUINT32)((gctUINT32_PTR)PageTable - area->stlbLogical); |
| |
| node = area->mapLogical + index; |
| |
| if (pageCount != _GetPageCountOfUsedNode(node)) |
| { |
| gcmkONERROR(gcvSTATUS_INVALID_REQUEST); |
| } |
| |
| #if gcdBOUNDARY_CHECK |
| node -= gcdBOUNDARY_CHECK; |
| #endif |
| |
| gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| if (Mmu->hardware->mmuVersion == 0) |
| { |
| _FillPageTable(PageTable, pageCount, Mmu->safeAddress); |
| } |
| |
| if (pageCount == 1) |
| { |
| /* Single page node. */ |
| node[0] = (~((1U<<8)-1)) | gcvMMU_SINGLE; |
| |
| if (PageTable != gcvNULL) |
| { |
| #if gcdUSE_MMU_EXCEPTION |
| /* Enable exception */ |
| _WritePageEntry(PageTable, (1 << 1)); |
| #else |
| _WritePageEntry(PageTable, 0); |
| #endif |
| } |
| } |
| else |
| { |
| /* Mark the node as free. */ |
| node[0] = (pageCount << 8) | gcvMMU_FREE; |
| node[1] = ~0U; |
| |
| if (PageTable != gcvNULL) |
| { |
| #if gcdUSE_MMU_EXCEPTION |
| /* Enable exception */ |
| gcmkVERIFY_OK(_FillPageTable(PageTable, (gctUINT32)PageCount, 1 << 1)); |
| #else |
| gcmkVERIFY_OK(_FillPageTable(PageTable, (gctUINT32)PageCount, 0)); |
| #endif |
| } |
| } |
| |
| gcmkDUMP(Mmu->os, |
| "#[mmu-stlb: free 0x%08X - 0x%08X]", |
| Address, Address + pageCount * pageSize - 1); |
| |
| gcmkDUMP(Mmu->os, |
| "@[physical.fill 0x%010llX 0x%08X 0x%08X]", |
| (unsigned long long)(area->stlbPhysical + index * 4), |
| *(gctUINT32_PTR)PageTable, |
| pageCount * 4); |
| |
| /* We have free nodes. */ |
| area->freeNodes = gcvTRUE; |
| |
| /* Record freed address range. */ |
| data.addressData.start = Address; |
| data.addressData.end = Address + (gctUINT32)PageCount * pageSize; |
| gckQUEUE_Enqueue(&Mmu->recentFreedAddresses, &data); |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); |
| acquired = gcvFALSE; |
| |
| #if gcdENABLE_TRUST_APPLICATION |
| if (Mmu->hardware->options.secureMode == gcvSECURE_IN_TA) |
| { |
| gckKERNEL_SecurityUnmapMemory(Mmu->hardware->kernel, Address, (gctUINT32)PageCount); |
| } |
| #endif |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckMMU_AllocatePages( |
| IN gckMMU Mmu, |
| IN gctSIZE_T PageCount, |
| IN gcePAGE_TYPE PageType, |
| OUT gctPOINTER * PageTable, |
| OUT gctUINT32 * Address |
| ) |
| { |
| return gckMMU_AllocatePagesEx( |
| Mmu, PageCount, gcvVIDMEM_TYPE_GENERIC, PageType, gcvFALSE, PageTable, Address); |
| } |
| |
| gceSTATUS |
| gckMMU_AllocatePagesEx( |
| IN gckMMU Mmu, |
| IN gctSIZE_T PageCount, |
| IN gceVIDMEM_TYPE Type, |
| IN gcePAGE_TYPE PageType, |
| IN gctBOOL Secure, |
| OUT gctPOINTER * PageTable, |
| OUT gctUINT32 * Address |
| ) |
| { |
| #if gcdDISABLE_GPU_VIRTUAL_ADDRESS |
| gcmkPRINT("GPU virtual address is disabled."); |
| return gcvSTATUS_NOT_SUPPORTED; |
| #else |
| return _AllocatePages(Mmu, PageCount, Type, PageType, Secure, PageTable, Address); |
| #endif |
| } |
| |
| gceSTATUS |
| gckMMU_FreePages( |
| IN gckMMU Mmu, |
| IN gctBOOL Secure, |
| IN gcePAGE_TYPE PageType, |
| IN gctUINT32 Address, |
| IN gctPOINTER PageTable, |
| IN gctSIZE_T PageCount |
| ) |
| { |
| return _FreePages(Mmu, Secure, PageType, Address, PageTable, PageCount); |
| } |
| |
| gceSTATUS |
| gckMMU_SetPage( |
| IN gckMMU Mmu, |
| IN gctPHYS_ADDR_T PageAddress, |
| IN gcePAGE_TYPE PageType, |
| IN gctBOOL Writable, |
| IN gctUINT32 *PageEntry |
| ) |
| { |
| gctUINT32 addressExt; |
| gctUINT32 address; |
| |
| gcmkHEADER_ARG("Mmu=0x%x", Mmu); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); |
| gcmkVERIFY_ARGUMENT(PageEntry != gcvNULL); |
| |
| if (PageType == gcvPAGE_TYPE_1M) |
| { |
| gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFFFF)); |
| } |
| else |
| { |
| gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFF)); |
| } |
| |
| /* [31:0]. */ |
| address = (gctUINT32)(PageAddress & 0xFFFFFFFF); |
| /* [39:32]. */ |
| addressExt = (gctUINT32)((PageAddress >> 32) & 0xFF); |
| |
| if (Mmu->hardware->mmuVersion == 0) |
| { |
| _WritePageEntry(PageEntry, address); |
| } |
| else |
| { |
| _WritePageEntry(PageEntry, _SetPage(address, addressExt, gcvTRUE)); |
| } |
| |
| #ifdef DUMP_IN_KERNEL |
| { |
| gctUINT32 *stlbLogical = (PageType == gcvPAGE_TYPE_1M) |
| ? Mmu->dynamicArea1M.stlbLogical |
| : Mmu->dynamicArea4K.stlbLogical; |
| |
| gctPHYS_ADDR_T stlbPhysical = (PageType == gcvPAGE_TYPE_1M) |
| ? Mmu->dynamicArea1M.stlbPhysical |
| : Mmu->dynamicArea4K.stlbPhysical; |
| |
| gctPHYS_ADDR_T physical = stlbPhysical + (stlbLogical - PageEntry) * 4; |
| |
| gckDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 4", |
| physical, *PageEntry); |
| } |
| #endif |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckMMU_Flush( |
| IN gckMMU Mmu, |
| IN gceVIDMEM_TYPE Type |
| ) |
| { |
| gckHARDWARE hardware; |
| gctUINT32 mask; |
| gctINT i; |
| |
| if (Type == gcvVIDMEM_TYPE_VERTEX_BUFFER || |
| Type == gcvVIDMEM_TYPE_INDEX_BUFFER || |
| Type == gcvVIDMEM_TYPE_COMMAND) |
| { |
| mask = gcvPAGE_TABLE_DIRTY_BIT_FE; |
| } |
| else |
| { |
| mask = gcvPAGE_TABLE_DIRTY_BIT_OTHER; |
| } |
| |
| #if gcdSHARED_PAGETABLE |
| for (i = 0; i < gcdMAX_GPU_COUNT; i++) |
| { |
| gctUINT j; |
| hardware = sharedPageTable->hardwares[i]; |
| if (hardware) |
| { |
| for (j = 0; j < gcvENGINE_GPU_ENGINE_COUNT; j++) |
| { |
| gcmkVERIFY_OK(gckOS_AtomSetMask(hardware->pageTableDirty[j], mask)); |
| } |
| } |
| } |
| #else |
| hardware = Mmu->hardware; |
| |
| for (i = 0 ; i < gcvENGINE_GPU_ENGINE_COUNT; i++) |
| { |
| gcmkVERIFY_OK( |
| gckOS_AtomSetMask(hardware->pageTableDirty[i], mask)); |
| } |
| |
| { |
| gcsLISTHEAD_PTR hardwareHead; |
| gcmkLIST_FOR_EACH(hardwareHead, &Mmu->hardwareList) |
| { |
| hardware = gcmCONTAINEROF(hardwareHead, struct _gckHARDWARE, mmuHead); |
| |
| if (hardware != Mmu->hardware) |
| { |
| for (i = 0 ; i < gcvENGINE_GPU_ENGINE_COUNT; i++) |
| { |
| gcmkVERIFY_OK( |
| gckOS_AtomSetMask(hardware->pageTableDirty[i], mask)); |
| } |
| } |
| } |
| } |
| #endif |
| |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckMMU_DumpPageTableEntry( |
| IN gckMMU Mmu, |
| IN gceAREA_TYPE AreaType, |
| IN gctUINT32 Address |
| ) |
| { |
| gctUINT32_PTR pageTable; |
| gctUINT32 index; |
| gctUINT32 mtlb, stlb; |
| gcsADDRESS_AREA_PTR area = (AreaType == gcvAREA_TYPE_4K) |
| ? &Mmu->dynamicArea4K : &Mmu->dynamicArea1M; |
| |
| gctUINT32 stlbShift = (AreaType == gcvAREA_TYPE_4K) |
| ? gcdMMU_STLB_4K_SHIFT : gcdMMU_STLB_1M_SHIFT; |
| |
| gctUINT32 stlbMask = (AreaType == gcvAREA_TYPE_4K) |
| ? gcdMMU_STLB_4K_MASK : gcdMMU_STLB_1M_MASK; |
| |
| gctUINT32 stlbEntryNum = (AreaType == gcvAREA_TYPE_4K) |
| ? gcdMMU_STLB_4K_ENTRY_NUM : gcdMMU_STLB_1M_ENTRY_NUM; |
| |
| gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address); |
| gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); |
| |
| gcmkASSERT(Mmu->hardware->mmuVersion > 0); |
| |
| mtlb = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; |
| |
| if (AreaType != gcvAREA_TYPE_FLATMAP) |
| { |
| stlb = (Address & stlbMask) >> stlbShift; |
| |
| pageTable = area->stlbLogical; |
| |
| index = (mtlb - area->mappingStart) |
| * stlbEntryNum |
| + stlb; |
| |
| gcmkPRINT(" Page table entry = 0x%08X", _ReadPageEntry(pageTable + index)); |
| } |
| else |
| { |
| gcsMMU_STLB_CHUNK_PTR stlbChunkObj = Mmu->staticSTLB; |
| gctUINT32 entry = Mmu->mtlbLogical[mtlb]; |
| |
| stlb = (Address & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; |
| |
| entry &= 0xFFFFFFF0; |
| |
| while (stlbChunkObj) |
| { |
| gctUINT i; |
| gctBOOL found = gcvFALSE; |
| for (i = 0; i < stlbChunkObj->mtlbEntryNum; ++i) |
| { |
| gctPHYS_ADDR_T stlbPhysBase = stlbChunkObj->physBase + (i * gcdMMU_STLB_1M_SIZE); |
| gctUINT32_PTR stlbLogical = |
| (gctUINT32_PTR)((gctUINT8_PTR)stlbChunkObj->logical + (i * gcdMMU_STLB_1M_SIZE)); |
| if (entry == stlbPhysBase) |
| { |
| gcmkPRINT(" Page table entry = 0x%08X", stlbLogical[stlb]); |
| found = gcvTRUE; |
| break; |
| } |
| } |
| if (found) |
| break; |
| stlbChunkObj = stlbChunkObj->next; |
| } |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| void |
| gckMMU_CheckSaftPage( |
| IN gckMMU Mmu |
| ) |
| { |
| gctUINT8_PTR safeLogical = Mmu->safePageLogical; |
| gctUINT32 offsets[] = { |
| 0, |
| 64, |
| 128, |
| 256, |
| 2560, |
| 4000 |
| }; |
| |
| gctUINT32 i = 0; |
| |
| while (i < gcmCOUNTOF(offsets)) |
| { |
| if (safeLogical[offsets[i]] != 0) |
| { |
| gcmkPRINT("%s(%d) safe page is over written [%d] = %x", |
| __FUNCTION__, __LINE__, i, safeLogical[offsets[i]]); |
| } |
| } |
| } |
| |
| void |
| gckMMU_DumpAddressSpace( |
| IN gckMMU Mmu |
| ) |
| { |
| gctUINT i; |
| gctUINT next; |
| /* TODO: */ |
| gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K; |
| gctUINT32_PTR map = area->mapLogical; |
| gctBOOL used = gcvFALSE; |
| gctUINT32 numPages; |
| |
| /* Grab the mutex. */ |
| gcmkVERIFY_OK(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); |
| |
| /* Find node which contains index. */ |
| for (i = 0; i < area->stlbEntries; i = next) |
| { |
| switch (gcmENTRY_TYPE(map[i])) |
| { |
| case gcvMMU_SINGLE: |
| numPages = 1; |
| next = i + numPages; |
| used = gcvFALSE; |
| break; |
| |
| case gcvMMU_FREE: |
| numPages = map[i] >> 8; |
| next = i + numPages; |
| used = gcvFALSE; |
| break; |
| |
| case gcvMMU_USED: |
| numPages = 1; |
| next = i + numPages; |
| used = gcvTRUE; |
| break; |
| |
| default: |
| gcmkFATAL("MMU table correcupted at index %u!", i); |
| return; |
| } |
| |
| if (!used) |
| { |
| gcmkPRINT("Available Range [%d - %d)", i, i + numPages); |
| } |
| } |
| |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); |
| |
| } |
| |
| void |
| gckMMU_DumpRecentFreedAddress( |
| IN gckMMU Mmu |
| ) |
| { |
| gckQUEUE queue = &Mmu->recentFreedAddresses; |
| gctUINT32 i; |
| gcuQUEUEDATA *data; |
| |
| if (queue->count) |
| { |
| gcmkPRINT(" Recent %d freed GPU address ranges:", queue->count); |
| |
| for (i = 0; i < queue->count; i++) |
| { |
| gckQUEUE_GetData(queue, i, &data); |
| |
| gcmkPRINT(" [%08X - %08X]", data->addressData.start, data->addressData.end); |
| } |
| } |
| } |
| |
| gceSTATUS |
| gckMMU_FillFlatMapping( |
| IN gckMMU Mmu, |
| IN gctUINT64 PhysBase, |
| IN gctSIZE_T Size |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware = Mmu->hardware; |
| |
| if (hardware->mmuVersion) |
| { |
| gcmkONERROR(_FillFlatMapping(Mmu, PhysBase, Size, gcvFALSE, gcvTRUE, gcvNULL)); |
| } |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| return status; |
| } |
| |
| gceSTATUS |
| gckMMU_IsFlatMapped( |
| IN gckMMU Mmu, |
| IN gctUINT64 Physical, |
| IN gctUINT32 Address, |
| OUT gctBOOL *In |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 i; |
| gctBOOL inFlatmapping = gcvFALSE; |
| |
| gcmkHEADER(); |
| |
| gcmkVERIFY_ARGUMENT(In != gcvNULL); |
| |
| if (gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_MMU) == gcvFALSE) |
| { |
| gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); |
| } |
| |
| if (Physical != gcvINVALID_PHYSICAL_ADDRESS) |
| { |
| for (i = 0; i < Mmu->gpuPhysicalRangeCount; i++) |
| { |
| if ((Physical >= Mmu->gpuPhysicalRanges[i].start) && |
| (Physical <= Mmu->gpuPhysicalRanges[i].end)) |
| { |
| inFlatmapping = gcvTRUE; |
| break; |
| } |
| } |
| } |
| |
| if (Address != gcvINVALID_ADDRESS) |
| { |
| for (i = 0; i < Mmu->gpuAddressRangeCount; i++) |
| { |
| if ((Address >= Mmu->gpuAddressRanges[i].start) && |
| (Address <= Mmu->gpuAddressRanges[i].end)) |
| { |
| inFlatmapping = gcvTRUE; |
| break; |
| } |
| } |
| } |
| |
| *In = inFlatmapping; |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckMMU_SetupSRAM( |
| IN gckMMU Mmu, |
| IN gckHARDWARE Hardware, |
| IN gckDEVICE Device |
| ) |
| { |
| gctBOOL needMapInternalSRAM = gcvFALSE; |
| gctPHYS_ADDR_T reservedBase = gcvINVALID_PHYSICAL_ADDRESS; |
| gctUINT32 reservedSize = 0; |
| gctUINT i = 0; |
| gctUINT j = 0; |
| gceSTATUS status; |
| gckKERNEL kernel = Hardware->kernel; |
| |
| gcmkHEADER_ARG("Mmu=0x%x Hardware=0x%x", Mmu, Hardware); |
| |
| gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); |
| |
| if (Hardware->mmuVersion == 0) |
| { |
| gcmkONERROR(gcvSTATUS_OK); |
| } |
| |
| if (!Mmu->sRAMMapped) |
| { |
| gctUINT32 address = gcvINVALID_ADDRESS; |
| gctUINT32 size = 0; |
| |
| /* Map all the SRAMs in MMU table. */ |
| for (i = 0; i < gcvCORE_COUNT; i++) |
| { |
| for (j = gcvSRAM_INTERNAL0; j < gcvSRAM_INTER_COUNT; j++) |
| { |
| reservedBase = Device->sRAMBases[i][j]; |
| reservedSize = Device->sRAMSizes[i][j]; |
| |
| Device->sRAMBaseAddresses[i][j] = 0; |
| |
| needMapInternalSRAM = reservedSize && (reservedBase != gcvINVALID_PHYSICAL_ADDRESS); |
| |
| /* Map the internal SRAM. */ |
| if (needMapInternalSRAM) |
| { |
| if (Device->showSRAMMapInfo) |
| { |
| gcmkPRINT("Galcore Info: MMU mapped core%d SRAM base=0x%llx size=0x%x", |
| i, |
| reservedBase, |
| reservedSize |
| ); |
| } |
| |
| /* |
| * Default gpu virtual base = 0. |
| * It can be specified if not conflict with existing mapping. |
| */ |
| gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical( |
| Mmu->os, |
| reservedBase, |
| &reservedBase |
| )); |
| |
| gcmkONERROR(_FillFlatMapping( |
| Mmu, |
| reservedBase, |
| reservedSize, |
| gcvTRUE, |
| gcvTRUE, |
| &Device->sRAMBaseAddresses[i][j] |
| )); |
| |
| Device->sRAMBases[i][j] = reservedBase; |
| } |
| else if (reservedSize && reservedBase == gcvINVALID_PHYSICAL_ADDRESS) |
| { |
| /* |
| * Reserve the internal SRAM range in first reserved MMU mtlb, |
| * when CPU physical base address is not specified, which means it is reserve usage. |
| */ |
| Device->sRAMBaseAddresses[i][j] = (address == gcvINVALID_ADDRESS) ? gcdRESERVE_START : |
| address + gcmALIGN(size, gcdRESERVE_ALIGN); |
| |
| Device->sRAMBases[i][j] = address = Device->sRAMBaseAddresses[i][j]; |
| |
| size = Device->sRAMSizes[i][j]; |
| |
| Device->sRAMPhysFaked[i][j] = gcvTRUE; |
| } |
| |
| #if gcdCAPTURE_ONLY_MODE |
| Device->sRAMPhysFaked[i][j] = gcvTRUE; |
| #endif |
| } |
| } |
| |
| /* Map all the external SRAMs in MMU table. */ |
| for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++) |
| { |
| if (Device->extSRAMSizes[i] && |
| (Device->extSRAMBases[i] != gcvINVALID_PHYSICAL_ADDRESS)) |
| { |
| |
| Device->extSRAMBaseAddresses[i] = 0; |
| |
| gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical( |
| Mmu->os, |
| Device->extSRAMBases[i], |
| &Device->extSRAMGPUBases[i] |
| )); |
| |
| gcmkONERROR(_FillFlatMapping( |
| Mmu, |
| Device->extSRAMGPUBases[i], |
| Device->extSRAMSizes[i], |
| gcvFALSE, |
| gcvTRUE, |
| &Device->extSRAMBaseAddresses[i] |
| )); |
| |
| Device->extSRAMGPUPhysNames[i] = gckKERNEL_AllocateNameFromPointer(kernel, Device->extSRAMPhysical[i]); |
| } |
| } |
| Mmu->sRAMMapped = gcvTRUE; |
| } |
| |
| /* Get per core SRAM hardware base address. */ |
| for (i = gcvSRAM_INTERNAL0; i < gcvSRAM_INTER_COUNT; i++) |
| { |
| if (Device->sRAMSizes[Hardware->core][i] && |
| (Device->sRAMBaseAddresses[Hardware->core][i])) |
| { |
| kernel->sRAMBaseAddresses[i] = Device->sRAMBaseAddresses[Hardware->core][i]; |
| kernel->sRAMSizes[i] = Hardware->options.sRAMSizes[i] |
| = Device->sRAMSizes[Hardware->core][i]; |
| |
| kernel->sRAMPhysFaked[i] = Device->sRAMPhysFaked[Hardware->core][i]; |
| Hardware->options.sRAMGPUVirtAddrs[i] = Device->sRAMBaseAddresses[Hardware->core][i]; |
| |
| /* If the internal SRAM usage is reserve. */ |
| if (kernel->sRAMPhysFaked[i]) |
| { |
| /* Use virtual address as the faked physical address which will never be accessed. */ |
| status = gckVIDMEM_Construct( |
| Mmu->os, |
| (gctPHYS_ADDR_T)kernel->sRAMBaseAddresses[i], |
| kernel->sRAMSizes[i], |
| 64, |
| 0, |
| &kernel->sRAMVidMem[i] |
| ); |
| |
| if (gcmIS_ERROR(status)) |
| { |
| kernel->sRAMSizes[i] = 0; |
| kernel->sRAMVidMem[i] = gcvNULL; |
| } |
| else |
| { |
| gcmkONERROR(gckOS_RequestReservedMemory( |
| Mmu->os, |
| (gctPHYS_ADDR_T)kernel->sRAMBaseAddresses[i], |
| kernel->sRAMSizes[i], |
| "Per core SRAM reserve usage region", |
| gcvTRUE, |
| &kernel->sRAMPhysical[i] |
| )); |
| |
| kernel->sRAMVidMem[i]->physical = kernel->sRAMPhysical[i]; |
| } |
| } |
| |
| if (Device->showSRAMMapInfo) |
| { |
| gcmkPRINT("Galcore Info: MMU mapped core %d SRAM[%d] hardware virtual address=0x%x size=0x%x", |
| Hardware->core, |
| i, |
| kernel->sRAMBaseAddresses[i], |
| kernel->sRAMSizes[i] |
| ); |
| } |
| } |
| } |
| |
| for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++) |
| { |
| if (Device->extSRAMSizes[i] && |
| (Device->extSRAMBases[i] != gcvINVALID_PHYSICAL_ADDRESS)) |
| { |
| kernel->extSRAMBaseAddresses[i] = Device->extSRAMBaseAddresses[i]; |
| |
| Hardware->options.extSRAMGPUVirtAddrs[i] = Device->extSRAMBaseAddresses[i]; |
| Hardware->options.extSRAMSizes[i] = Device->extSRAMSizes[i]; |
| Hardware->options.extSRAMCPUPhysAddrs[i] = Device->extSRAMBases[i]; |
| Hardware->options.extSRAMGPUPhysAddrs[i] = Device->extSRAMGPUBases[i]; |
| Hardware->options.extSRAMGPUPhysNames[i] = Device->extSRAMGPUPhysNames[i]; |
| |
| if (Device->showSRAMMapInfo) |
| { |
| gcmkPRINT("Galcore Info: MMU mapped external shared SRAM[%d] CPU view base=0x%llx GPU view base=0x%llx GPU virtual address=0x%x size=0x%x", |
| i, |
| Device->extSRAMBases[i], |
| Device->extSRAMGPUBases[i], |
| kernel->extSRAMBaseAddresses[i], |
| Device->extSRAMSizes[i] |
| ); |
| } |
| } |
| } |
| |
| gcsLIST_Add(&Hardware->mmuHead, &Mmu->hardwareList); |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the error. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| |
| gceSTATUS |
| gckMMU_GetPageEntry( |
| IN gckMMU Mmu, |
| IN gcePAGE_TYPE PageType, |
| IN gctUINT32 Address, |
| IN gctUINT32_PTR *PageTable |
| ) |
| { |
| gctUINT32_PTR pageTable; |
| gctUINT32 index; |
| gctUINT32 mtlbOffset, stlbOffset; |
| gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, gcvFALSE); |
| |
| gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address); |
| gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); |
| |
| gcmkASSERT(Mmu->hardware->mmuVersion > 0); |
| |
| mtlbOffset = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; |
| |
| if (mtlbOffset >= area->mappingStart) |
| { |
| gctUINT32 stlbShift = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT; |
| gctUINT32 stlbMask = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_MASK : gcdMMU_STLB_4K_MASK; |
| gctUINT32 stlbEntryNum = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM; |
| |
| stlbOffset = (Address & stlbMask) >> stlbShift; |
| |
| pageTable = area->stlbLogical; |
| |
| index = (mtlbOffset - area->mappingStart) * stlbEntryNum + stlbOffset; |
| |
| *PageTable = pageTable + index; |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckMMU_GetAreaType( |
| IN gckMMU Mmu, |
| IN gctUINT32 GpuAddress, |
| OUT gceAREA_TYPE *AreaType |
| ) |
| { |
| gctUINT32 mtlbIndex; |
| gctBOOL flatMapped; |
| gceSTATUS status = gcvSTATUS_OK; |
| gcsADDRESS_AREA_PTR area4K = &Mmu->dynamicArea4K; |
| #if gcdENABLE_GPU_1M_PAGE |
| gcsADDRESS_AREA_PTR area1M = &Mmu->dynamicArea1M; |
| #endif |
| |
| mtlbIndex = _MtlbOffset(GpuAddress); |
| |
| gcmkONERROR(gckMMU_IsFlatMapped(Mmu, gcvINVALID_PHYSICAL_ADDRESS, GpuAddress, &flatMapped)); |
| |
| if (flatMapped) |
| { |
| *AreaType = gcvAREA_TYPE_FLATMAP; |
| } |
| #if gcdENABLE_GPU_1M_PAGE |
| else if (mtlbIndex >= area1M->mappingStart && mtlbIndex <= area1M->mappingEnd) |
| { |
| *AreaType = gcvAREA_TYPE_1M; |
| } |
| #endif |
| else if (mtlbIndex >= area4K->mappingStart && mtlbIndex <= area4K->mappingEnd) |
| { |
| *AreaType = gcvAREA_TYPE_4K; |
| } |
| else |
| { |
| *AreaType = gcvAREA_TYPE_UNKNOWN; |
| } |
| |
| OnError: |
| return status; |
| } |