Google Git
Sign in
coral / imx-gpu-viv-ko / 8aa0d455633b441846b161574622278782bb71c0 / . / hal / kernel / gc_hal_kernel_mmu.c
blob: 714ff28dedd01c4dca0030192ec75d1e7d8a87ba [file] [log] [blame]
/****************************************************************************
*
* The MIT License (MIT)
*
* Copyright (c) 2014 - 2020 Vivante Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
*****************************************************************************
*
* The GPL License (GPL)
*
* Copyright (C) 2014 - 2020 Vivante Corporation
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*****************************************************************************
*
* Note: This software is released under dual MIT and GPL licenses. A
* recipient may use this file under the terms of either the MIT license or
* GPL License. If you wish to use only one license not the other, you can
* indicate your decision by deleting one of the above license notices in your
* version of this file.
*
*****************************************************************************/
#include "gc_hal_kernel_precomp.h"
#define _GC_OBJ_ZONE gcvZONE_MMU
typedef enum _gceMMU_TYPE
{
gcvMMU_USED = (0 << 4),
gcvMMU_SINGLE = (1 << 4),
gcvMMU_FREE = (2 << 4),
}
gceMMU_TYPE;
/* VIP SRAM start virtual address. */
#define gcdRESERVE_START (4 << 20)
#define gcdRESERVE_ALIGN (4 << 10)
#define gcmENTRY_TYPE(x) (x & 0xF0)
#define gcmENTRY_COUNT(x) ((x & 0xFFFFFF00) >> 8)
#define gcdMMU_TABLE_DUMP 0
#define gcdVERTEX_START (128 << 10)
typedef struct _gcsMMU_STLB_CHUNK *gcsMMU_STLB_CHUNK_PTR;
typedef struct _gcsMMU_STLB_CHUNK
{
gckVIDMEM_NODE videoMem;
gctUINT32_PTR logical;
gctSIZE_T size;
gctPHYS_ADDR_T physBase;
gctSIZE_T pageCount;
gctUINT32 mtlbIndex;
gctUINT32 mtlbEntryNum;
gcsMMU_STLB_CHUNK_PTR next;
} gcsMMU_STLB_CHUNK;
#if gcdSHARED_PAGETABLE
typedef struct _gcsSharedPageTable * gcsSharedPageTable_PTR;
typedef struct _gcsSharedPageTable
{
/* Shared gckMMU object. */
gckMMU mmu;
/* Hardwares which use this shared pagetable. */
gckHARDWARE hardwares[gcdMAX_GPU_COUNT];
/* Number of cores use this shared pagetable. */
gctUINT32 reference;
}
gcsSharedPageTable;
static gcsSharedPageTable_PTR sharedPageTable = gcvNULL;
#endif
typedef struct _gcsFreeSpaceNode * gcsFreeSpaceNode_PTR;
typedef struct _gcsFreeSpaceNode
{
gctUINT32 start;
gctINT32 entries;
}
gcsFreeSpaceNode;
#if gcdENDIAN_BIG
# define _WritePageEntry(pageEntry, entryValue) \
*(gctUINT32_PTR)(pageEntry) = gcmBSWAP32((gctUINT32)(entryValue))
# define _ReadPageEntry(pageEntry) \
gcmBSWAP32(*(gctUINT32_PTR)(pageEntry))
#else
# define _WritePageEntry(pageEntry, entryValue) \
*(gctUINT32_PTR)(pageEntry) = (gctUINT32)(entryValue)
# define _ReadPageEntry(pageEntry) \
*(gctUINT32_PTR)(pageEntry)
#endif
static gceSTATUS
_FillPageTable(
IN gctUINT32_PTR PageTable,
IN gctUINT32 PageCount,
IN gctUINT32 EntryValue
)
{
gctUINT i;
for (i = 0; i < PageCount; i++)
{
_WritePageEntry(PageTable + i, EntryValue);
}
return gcvSTATUS_OK;
}
static gceSTATUS
_FillMap(
IN gctUINT32_PTR Map,
IN gctUINT32 PageCount,
IN gctUINT32 EntryValue
)
{
gctUINT i;
for (i = 0; i < PageCount; i++)
{
Map[i] = EntryValue;
}
return gcvSTATUS_OK;
}
static gceSTATUS
_Link(
IN gcsADDRESS_AREA_PTR Area,
IN gctUINT32 Index,
IN gctUINT32 Next
)
{
if (Index >= Area->stlbEntries)
{
/* Just move heap pointer. */
Area->heapList = Next;
}
else
{
/* Address page table. */
gctUINT32_PTR map = Area->mapLogical;
/* Dispatch on node type. */
switch (gcmENTRY_TYPE(map[Index]))
{
case gcvMMU_SINGLE:
/* Set single index. */
map[Index] = (Next << 8) | gcvMMU_SINGLE;
break;
case gcvMMU_FREE:
/* Set index. */
map[Index + 1] = Next;
break;
default:
gcmkFATAL("MMU table correcupted at index %u!", Index);
return gcvSTATUS_HEAP_CORRUPTED;
}
}
/* Success. */
return gcvSTATUS_OK;
}
static gceSTATUS
_AddFree(
IN gcsADDRESS_AREA_PTR Area,
IN gctUINT32 Index,
IN gctUINT32 Node,
IN gctUINT32 Count
)
{
gctUINT32_PTR map = Area->mapLogical;
if (Count == 1)
{
/* Initialize a single page node. */
map[Node] = (~((1U<<8)-1)) | gcvMMU_SINGLE;
}
else
{
/* Initialize the node. */
map[Node + 0] = (Count << 8) | gcvMMU_FREE;
map[Node + 1] = ~0U;
}
/* Append the node. */
return _Link(Area, Index, Node);
}
static gceSTATUS
_Collect(
IN gcsADDRESS_AREA_PTR Area
)
{
gctUINT32_PTR map = Area->mapLogical;
gceSTATUS status;
gctUINT32 i, previous, start = 0, count = 0;
previous = Area->heapList = ~0U;
Area->freeNodes = gcvFALSE;
/* Walk the entire page table. */
for (i = 0; i < Area->stlbEntries; ++i)
{
/* Dispatch based on type of page. */
switch (gcmENTRY_TYPE(map[i]))
{
case gcvMMU_USED:
/* Used page, so close any open node. */
if (count > 0)
{
/* Add the node. */
gcmkONERROR(_AddFree(Area, previous, start, count));
/* Reset the node. */
previous = start;
count = 0;
}
break;
case gcvMMU_SINGLE:
/* Single free node. */
if (count++ == 0)
{
/* Start a new node. */
start = i;
}
break;
case gcvMMU_FREE:
/* A free node. */
if (count == 0)
{
/* Start a new node. */
start = i;
}
/* Advance the count. */
count += map[i] >> 8;
/* Advance the index into the page table. */
i += (map[i] >> 8) - 1;
break;
default:
gcmkFATAL("MMU page table correcupted at index %u!", i);
return gcvSTATUS_HEAP_CORRUPTED;
}
}
/* See if we have an open node left. */
if (count > 0)
{
/* Add the node to the list. */
gcmkONERROR(_AddFree(Area, previous, start, count));
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_MMU,
"Performed a garbage collection of the MMU heap.");
/* Success. */
return gcvSTATUS_OK;
OnError:
/* Return the staus. */
return status;
}
static gctUINT32
_SetPage(gctUINT32 PageAddress, gctUINT32 PageAddressExt, gctBOOL Writable)
{
gctUINT32 entry = PageAddress
/* AddressExt */
| (PageAddressExt << 4)
/* Ignore exception */
| (0 << 1)
/* Present */
| (1 << 0);
if (Writable)
{
/* writable */
entry |= (1 << 2);
}
#if gcdUSE_MMU_EXCEPTION
else
{
/* If this page is read only, set exception bit to make exception happens
** when writing to it. */
entry |= gcdMMU_STLB_EXCEPTION;
}
#endif
return entry;
}
static gctUINT32
_MtlbOffset(
gctUINT32 Address
)
{
return (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT;
}
gctUINT32
_AddressToIndex(
IN gcsADDRESS_AREA_PTR Area,
IN gctUINT32 Address
)
{
gctUINT32 stlbShift = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT;
gctUINT32 stlbMask = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_MASK : gcdMMU_STLB_4K_MASK;
gctUINT32 stlbEntryNum = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM;
gctUINT32 mtlbOffset = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT;
gctUINT32 stlbOffset = (Address & stlbMask) >> stlbShift;
return (mtlbOffset - Area->mappingStart) * stlbEntryNum + stlbOffset;
}
static gctUINT32_PTR
_StlbEntry(
gcsADDRESS_AREA_PTR Area,
gctUINT32 Address
)
{
gctUINT32 index = _AddressToIndex(Area, Address);
return &Area->stlbLogical[index];
}
static gctBOOL
_IsRangeInsected(
gctUINT64 baseAddress1,
gctSIZE_T size1,
gctUINT64 baseAddress2,
gctSIZE_T size2
)
{
gctUINT64 endAddress1 = baseAddress1 + size1 - 1;
gctUINT64 endAddress2 = baseAddress2 + size2 - 1;
if (!size1 || !size2)
{
return gcvFALSE;
}
return (((baseAddress2 <= endAddress1) && (endAddress2 >= baseAddress1)) ||
((baseAddress1 <= endAddress2) && (endAddress1 >= baseAddress2)));
}
static gceSTATUS
_FillFlatMappingInMap(
gcsADDRESS_AREA_PTR Area,
gctUINT32 Index,
gctUINT32 NumPages
)
{
gceSTATUS status;
gctUINT32 i;
gctBOOL gotIt = gcvFALSE;
gctUINT32 index = Index;
gctUINT32_PTR map = Area->mapLogical;
gctUINT32 previous = ~0U;
/* Find node which contains index. */
for (i = 0; !gotIt && (i < Area->stlbEntries);)
{
gctUINT32 numPages;
switch (gcmENTRY_TYPE(map[i]))
{
case gcvMMU_SINGLE:
if (i == index)
{
gotIt = gcvTRUE;
}
else
{
previous = i;
i = map[i] >> 8;
}
break;
case gcvMMU_FREE:
numPages = map[i] >> 8;
if (index >= i && index + NumPages - 1 < i + numPages)
{
gotIt = gcvTRUE;
}
else
{
previous = i;
i = map[i + 1];
}
break;
case gcvMMU_USED:
i++;
break;
default:
gcmkFATAL("MMU table correcupted at index %u!", index);
gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
}
}
switch (gcmENTRY_TYPE(map[i]))
{
case gcvMMU_SINGLE:
/* Unlink single node from free list. */
gcmkONERROR(
_Link(Area, previous, map[i] >> 8));
break;
case gcvMMU_FREE:
/* Split the node. */
{
gctUINT32 start;
gctUINT32 next = map[i+1];
gctUINT32 total = map[i] >> 8;
gctUINT32 countLeft = index - i;
gctUINT32 countRight = total - countLeft - NumPages;
if (countLeft)
{
start = i;
_AddFree(Area, previous, start, countLeft);
previous = start;
}
if (countRight)
{
start = index + NumPages;
_AddFree(Area, previous, start, countRight);
previous = start;
}
_Link(Area, previous, next);
}
break;
}
_FillMap(&map[index], NumPages, gcvMMU_USED);
return gcvSTATUS_OK;
OnError:
return status;
}
static gceSTATUS
_CollectFreeSpace(
IN gckMMU Mmu,
OUT gcsFreeSpaceNode_PTR *Array,
OUT gctINT * Size
)
{
gceSTATUS status = gcvSTATUS_OK;
gctPOINTER pointer = gcvNULL;
gcsFreeSpaceNode_PTR array = gcvNULL;
gcsFreeSpaceNode_PTR node = gcvNULL;
gctINT size = 0;
gctINT i = 0;
for (i = 0; i < gcdMMU_MTLB_ENTRY_NUM; i++)
{
if (!Mmu->mtlbLogical[i])
{
if (!node)
{
/* This is the first entry of the free space. */
node += 1;
size++;
}
}
else if (node)
{
/* Reset the start. */
node = gcvNULL;
}
}
/* Allocate memory for the array. */
gcmkONERROR(gckOS_Allocate(Mmu->os,
gcmSIZEOF(*array) * size,
&pointer));
array = (gcsFreeSpaceNode_PTR)pointer;
node = gcvNULL;
for (i = 0, size = 0; i < gcdMMU_MTLB_ENTRY_NUM; i++)
{
if (!Mmu->mtlbLogical[i])
{
if (!node)
{
/* This is the first entry of the free space. */
node = &array[size++];
node->start = i;
node->entries = 0;
}
node->entries++;
}
else if (node)
{
/* Reset the start. */
node = gcvNULL;
}
}
#if gcdMMU_TABLE_DUMP
for (i = 0; i < size; i++)
{
gckOS_Print("%s(%d): [%d]: start=%d, entries=%d.\n",
__FUNCTION__, __LINE__,
i,
array[i].start,
array[i].entries);
}
#endif
*Array = array;
*Size = size;
return gcvSTATUS_OK;
OnError:
if (pointer != gcvNULL)
{
gckOS_Free(Mmu->os, pointer);
}
return status;
}
gceSTATUS
_GetMtlbFreeSpace(
IN gckMMU Mmu,
IN gctUINT32 NumEntries,
OUT gctUINT32 *MtlbStart,
OUT gctUINT32 *MtlbEnd
)
{
gceSTATUS status = gcvSTATUS_OK;
gcsFreeSpaceNode_PTR nodeArray = gcvNULL;
gctINT i, nodeArraySize = 0;
gctINT numEntries = gcdMMU_MTLB_ENTRY_NUM;
gctINT32 mStart = -1;
gctINT32 mEnd = -1;
gcmkONERROR(_CollectFreeSpace(Mmu, &nodeArray, &nodeArraySize));
/* Find the smallest space for NumEntries */
for (i = 0; i < nodeArraySize; i++)
{
if (nodeArray[i].entries < numEntries && NumEntries <= (gctUINT32)nodeArray[i].entries)
{
numEntries = nodeArray[i].entries;
mStart = nodeArray[i].start;
mEnd = nodeArray[i].start + NumEntries - 1;
}
}
if (mStart == -1 && mEnd == -1)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
*MtlbStart = (gctUINT32)mStart;
*MtlbEnd = (gctUINT32)mEnd;
OnError:
if (nodeArray)
{
gckOS_Free(Mmu->os, (gctPOINTER)nodeArray);
}
return status;
}
static gcePOOL _GetPageTablePool(IN gckOS Os)
{
gcePOOL pool = gcvPOOL_DEFAULT;
gctUINT64 data = 0;
gceSTATUS status;
status = gckOS_QueryOption(Os, "mmuPageTablePool", &data);
if (status == gcvSTATUS_OK)
{
if (data == 1)
{
pool = gcvPOOL_VIRTUAL;
}
}
return pool;
}
static gceSTATUS
_FillFlatMapping(
IN gckMMU Mmu,
IN gctUINT64 PhysBase,
IN gctSIZE_T Size,
IN gctBOOL reserved,
IN gctBOOL ableToShift,
OUT gctUINT32 *GpuBaseAddress
)
{
gceSTATUS status;
gctUINT32 mtlb;
gctUINT32 physBase;
gckKERNEL kernel = Mmu->hardware->kernel;
gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K;
gctBOOL physicalRangeOverlapped = gcvFALSE;
gctBOOL virtualRangeOverlapped = gcvFALSE;
gctBOOL specificFlatMapping = gcvFALSE;
gctBOOL needShiftMapping = gcvFALSE;
gctUINT64 flatBase = PhysBase;
gctUINT32 flatSize = (gctUINT32) Size;
gctUINT64 base = flatBase;
gctUINT64 end = base + flatSize;
gctUINT32 reqVirtualBase = 0;
gctUINT32 flatVirtualBase = 0;
gctUINT32 i;
/************************ Get flat mapping type and range. ************************/
{
for (i = 0; i < Mmu->gpuPhysicalRangeCount; i++)
{
if (base < Mmu->gpuPhysicalRanges[i].start)
{
if (end > Mmu->gpuPhysicalRanges[i].start)
{
physicalRangeOverlapped = gcvTRUE;
if (Mmu->gpuPhysicalRanges[i].flag == gcvFLATMAP_DIRECT)
{
/* Overlapped part is direct mapping, continue direct mapping */
end = Mmu->gpuPhysicalRanges[i].start;
}
else
{
/* Overlapped part is shift mapping, do entire shift mapping */
needShiftMapping = gcvTRUE;
}
}
flatSize = (gctUINT32) (end - base);
}
else if (end > Mmu->gpuPhysicalRanges[i].end)
{
if (base < Mmu->gpuPhysicalRanges[i].end)
{
physicalRangeOverlapped = gcvTRUE;
if (Mmu->gpuPhysicalRanges[i].flag == gcvFLATMAP_DIRECT)
{
/* Overlapped part is direct mapping, continue direct mapping */
base = Mmu->gpuPhysicalRanges[i].end + 1;
}
else
{
/* Overlapped part is shift mapping, do entire shift mapping */
needShiftMapping = gcvTRUE;
}
}
flatBase = base;
flatSize = (gctUINT32) (end - base);
}
else
{
/* it is already inside existing flat mapping ranges. */
flatSize = 0;
}
if (flatSize == 0)
{
if (GpuBaseAddress)
{
*GpuBaseAddress = (gctUINT32) PhysBase;
}
return gcvSTATUS_OK;
}
}
}
/* overwrite the orignal parameters */
PhysBase = flatBase;
physBase = (gctUINT32)flatBase;
mtlb = _MtlbOffset(physBase);
if (GpuBaseAddress)
{
reqVirtualBase = *GpuBaseAddress;
}
/*
* if no partcial physical range overlap to request entire shift mapping,
* it is specific shift mapping or directly mapping by default.
*/
if (!needShiftMapping)
{
flatVirtualBase = reqVirtualBase ? reqVirtualBase : (gctUINT32)flatBase;
}
for (i = 0; i < Mmu->gpuAddressRangeCount; i++)
{
if (_IsRangeInsected(flatVirtualBase, flatSize,
Mmu->gpuAddressRanges[i].start, Mmu->gpuAddressRanges[i].size))
{
virtualRangeOverlapped = gcvTRUE;
}
}
/* If gpu virtual range overlapped or gpu physical over 4G, still need entire shift mapping */
if ((!physicalRangeOverlapped && virtualRangeOverlapped) ||
PhysBase + flatSize - 1 > 0xffffffff)
{
needShiftMapping = gcvTRUE;
}
if (needShiftMapping && !ableToShift)
{
/*
* Return without mapping any address.
* By now, only physBase physSize could run here.
*/
return gcvSTATUS_OK;
}
specificFlatMapping = (reqVirtualBase && !virtualRangeOverlapped && !physicalRangeOverlapped);
/************************ Setup flat mapping in dynamic range. ****************/
if (area->mappingStart != gcvINVALID_ADDRESS && mtlb >= area->mappingStart && mtlb < area->mappingEnd)
{
/* This path is useless now, keep it 4K page size */
gctUINT32_PTR stlbEntry;
stlbEntry = _StlbEntry(area, physBase);
/* Must be aligned to page. */
gcmkASSERT((flatSize & 0xFFF) == 0);
for (i = 0; i < (flatSize / gcdMMU_PAGE_4K_SIZE); i++)
{
/* Flat mapping in page table. */
_WritePageEntry(stlbEntry, _SetPage(physBase + i * gcdMMU_PAGE_4K_SIZE, 0, gcvTRUE));
#if gcdMMU_TABLE_DUMP
gckOS_Print("%s(%d): insert MTLB[%d] STLB[%d]: %08x\n",
__FUNCTION__, __LINE__,
(physBase & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT,
((physBase & gcdMMU_STLB_4K_MASK) >> gcdMMU_STLB_4K_SHIFT) + i,
_ReadPageEntry(stlbEntry));
#endif
stlbEntry++;
}
#if gcdDUMP_IN_KERNEL
{
gctPHYS_ADDR_T physical;
gctUINT32 data = _SetPage(physBase, 0, gcvTRUE) & ~0xF;
gctUINT32 step = (_SetPage(physBase + gcdMMU_PAGE_4K_SIZE, 0, gcvTRUE) & ~0xF) - data;
gctUINT32 mask = _SetPage(physBase, 0, gcvTRUE) & 0xF;
physical = area->stlbPhysical + 4 * _AddressToIndex(area, physBase);
gcmkDUMP(Mmu->os,
"#[mmu-stlb: flat-mapping in dynamic: 0x%08X - 0x%08X]",
physBase, physBase - 1 + flatSize);
gcmkDUMP(Mmu->os,
"@[physical.step 0x%010llX 0x%08X 0x%08lX 0x%08X 0x%08X",
(unsigned long long)physical, data,
(unsigned long)(flatSize / gcdMMU_PAGE_4K_SIZE * sizeof(gctUINT32)),
step, mask);
}
#endif
/* Flat mapping in map. */
_FillFlatMappingInMap(area, _AddressToIndex(area, physBase), flatSize / gcdMMU_PAGE_4K_SIZE);
return gcvSTATUS_OK;
}
/************************ Setup flat mapping in non dynamic range. **************/
{
gctBOOL mutex = gcvFALSE;
gctUINT32 physBaseExt = (gctUINT32) (PhysBase >> 32);
gctUINT32 start = physBase & ~gcdMMU_PAGE_1M_MASK;
gctUINT32 end = (gctUINT32) (physBase + flatSize - 1) & ~gcdMMU_PAGE_1M_MASK;
gctUINT32 mStart = start >> gcdMMU_MTLB_SHIFT;
gctUINT32 mEnd = end >> gcdMMU_MTLB_SHIFT;
gctUINT32 sStart = (start & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;
gctUINT32 sEnd = (end & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;
gctPHYS_ADDR_T physical;
gcsMMU_STLB_CHUNK_PTR newStlbChunk = gcvNULL;
gctUINT32 stlbIndex = 0;
gctUINT32 totalNewStlbs = 0;
gctINT32 firstMtlbEntry = -1;
gctUINT32 mtlbCurEntry;
gcsMMU_STLB_CHUNK_PTR curStlbChunk = gcvNULL;
gceFLATMAP_FLAG mapFlag = gcvFLATMAP_DIRECT;
enum
{
COLOR_NONE = 0,
COLOR_RED = 1, /* occupied entry */
COLOR_BLUE = 2, /* empty entry */
COLOR_MAX = COLOR_BLUE,
} lastColor = COLOR_NONE;
gctUINT32 colorNumber = 0;
/* Grab the mutex. */
gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
mutex = gcvTRUE;
if (needShiftMapping)
{
gctUINT32 mEntries;
gctUINT32 sEntries;
mEntries = (flatSize + (1 << gcdMMU_MTLB_SHIFT) - 1) / (1 << gcdMMU_MTLB_SHIFT);
gcmkONERROR(_GetMtlbFreeSpace(Mmu, mEntries, &mStart, &mEnd));
sStart = 0;
sEntries = (flatSize + gcdMMU_PAGE_1M_SIZE - 1) / gcdMMU_PAGE_1M_SIZE;
sEnd = (sEntries - 1) % gcdMMU_STLB_1M_ENTRY_NUM;
mapFlag = gcvFLATMAP_SHIFT;
}
if (specificFlatMapping)
{
start = reqVirtualBase & ~gcdMMU_PAGE_1M_MASK;
end = (reqVirtualBase + flatSize - 1) & ~gcdMMU_PAGE_1M_MASK;
mStart = start >> gcdMMU_MTLB_SHIFT;
mEnd = end >> gcdMMU_MTLB_SHIFT;
sStart = (start & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;
sEnd = (end & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;
mapFlag = gcvFLATMAP_SHIFT;
}
/* No matter direct mapping or shift mapping or specific mapping, store gpu virtual ranges */
flatVirtualBase = (mStart << gcdMMU_MTLB_SHIFT)
| (sStart << gcdMMU_STLB_1M_SHIFT)
| (physBase & gcdMMU_PAGE_1M_MASK);
/* Return GPU virtual base address if necessary */
if (GpuBaseAddress)
{
*GpuBaseAddress = flatVirtualBase;
}
mtlbCurEntry = mStart;
/* find all new stlbs, part of new flat mapping range may already have stlbs*/
while (mtlbCurEntry <= mEnd)
{
if (*(Mmu->mtlbLogical + mtlbCurEntry) == 0)
{
if (lastColor != COLOR_BLUE)
{
if (colorNumber < COLOR_MAX)
{
lastColor = COLOR_BLUE;
colorNumber++;
}
else
{
gcmkPRINT("There is a hole in new flat mapping range, which is not correct");
}
}
totalNewStlbs++;
if (-1 == firstMtlbEntry)
{
firstMtlbEntry = mtlbCurEntry;
}
}
else
{
if (lastColor != COLOR_RED)
{
if (colorNumber < COLOR_MAX)
{
lastColor = COLOR_RED;
colorNumber++;
}
else
{
gcmkPRINT("There is a hole in new flat mapping range, which is not correct");
}
}
}
mtlbCurEntry++;
}
/* Need allocate a new chunk of stlbs */
if (totalNewStlbs)
{
gcePOOL pool = Mmu->pool;
gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS;
gcmkONERROR(
gckOS_Allocate(Mmu->os,
sizeof(struct _gcsMMU_STLB_CHUNK),
(gctPOINTER *)&newStlbChunk));
newStlbChunk->mtlbEntryNum = totalNewStlbs;
newStlbChunk->next = gcvNULL;
newStlbChunk->videoMem = gcvNULL;
newStlbChunk->logical = gcvNULL;
newStlbChunk->size = gcdMMU_STLB_1M_SIZE * newStlbChunk->mtlbEntryNum;
newStlbChunk->pageCount = 0;
newStlbChunk->mtlbIndex = firstMtlbEntry;
#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
allocFlag |= gcvALLOC_FLAG_CACHEABLE;
#endif
allocFlag |= gcvALLOC_FLAG_4GB_ADDR;
gcmkONERROR(gckKERNEL_AllocateVideoMemory(
kernel,
gcdMMU_STLB_1M_SIZE,
gcvVIDMEM_TYPE_COMMAND,
allocFlag | gcvALLOC_FLAG_4K_PAGES,
&newStlbChunk->size,
&pool,
&newStlbChunk->videoMem));
/* Lock for kernel side CPU access. */
gcmkONERROR(gckVIDMEM_NODE_LockCPU(
kernel,
newStlbChunk->videoMem,
gcvFALSE,
gcvFALSE,
(gctPOINTER *)&newStlbChunk->logical));
gcmkONERROR(gckOS_ZeroMemory(newStlbChunk->logical, newStlbChunk->size));
/* Get CPU physical address. */
gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
kernel,
newStlbChunk->videoMem,
0,
&physical));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(
Mmu->os,
physical,
&physical));
newStlbChunk->physBase = physical;
}
while (mStart <= mEnd)
{
gctUINT32 last = (mStart == mEnd) ? sEnd : (gcdMMU_STLB_1M_ENTRY_NUM - 1);
gctPHYS_ADDR_T stlbPhyBase;
gctUINT32_PTR stlbLogical;
gcmkASSERT(mStart < gcdMMU_MTLB_ENTRY_NUM);
if (*(Mmu->mtlbLogical + mStart) == 0)
{
gctUINT32 mtlbEntry;
curStlbChunk = newStlbChunk;
stlbPhyBase = curStlbChunk->physBase + (stlbIndex * gcdMMU_STLB_1M_SIZE);
stlbLogical = (gctUINT32_PTR)((gctUINT8_PTR)curStlbChunk->logical + (stlbIndex * gcdMMU_STLB_1M_SIZE));
physical = stlbPhyBase
/* 1MB page size */
| (1 << 3)
/* Ignore exception */
| (0 << 1)
/* Present */
| (1 << 0);
gcmkSAFECASTPHYSADDRT(mtlbEntry, physical);
_WritePageEntry(Mmu->mtlbLogical + mStart, mtlbEntry);
#if gcdMMU_TABLE_DUMP
gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n",
__FUNCTION__, __LINE__,
mStart,
_ReadPageEntry(Mmu->mtlbLogical + mStart));
gckOS_Print("%s(%d): STLB: logical:%08x -> physical:%08x\n",
__FUNCTION__, __LINE__,
stlbLogical,
stlbPhyBase);
#endif
gcmkDUMP(Mmu->os, "#[mmu-mtlb: flat-mapping, slot: %d]", mStart);
gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08X]",
(unsigned long long)Mmu->mtlbPhysical + mStart * 4,
Mmu->mtlbLogical[mStart], 4);
++stlbIndex;
}
else
{
gctUINT32 mtlbEntry = _ReadPageEntry(Mmu->mtlbLogical + mStart);
gctUINT stlbOffset;
curStlbChunk = (gcsMMU_STLB_CHUNK_PTR)Mmu->staticSTLB;
while (curStlbChunk)
{
if ((mStart >= curStlbChunk->mtlbIndex) &&
(mStart < (curStlbChunk->mtlbIndex + curStlbChunk->mtlbEntryNum)))
{
break;
}
curStlbChunk = curStlbChunk->next;
}
gcmkASSERT(curStlbChunk);
if (!curStlbChunk)
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
stlbOffset = mStart - curStlbChunk->mtlbIndex;
stlbPhyBase = curStlbChunk->physBase + (stlbOffset * gcdMMU_STLB_1M_SIZE);
stlbLogical = (gctUINT32_PTR)((gctUINT8_PTR)curStlbChunk->logical + (stlbOffset * gcdMMU_STLB_1M_SIZE));
if (stlbPhyBase != (mtlbEntry & gcdMMU_MTLB_ENTRY_STLB_MASK))
{
gcmkASSERT(0);
}
}
#if gcdDUMP_IN_KERNEL
/* Start index. */
i = sStart;
gcmkDUMP(Mmu->os, "#[mmu-stlb: flat-mapping: 0x%08X - 0x%08X]",
start, start + (last - sStart) * gcdMMU_PAGE_1M_SIZE - 1);
#endif
while (sStart <= last)
{
gcmkASSERT(!(start & gcdMMU_PAGE_1M_MASK));
if (reserved)
{
/* program NOT_PRESENT | EXCEPTION for reserved entries */
_WritePageEntry(stlbLogical + sStart, 1 << 1);
}
else
{
_WritePageEntry(stlbLogical + sStart, _SetPage(start, physBaseExt, gcvTRUE));
}
#if gcdMMU_TABLE_DUMP
gckOS_Print("%s(%d): insert STLB[%d]: %08x\n",
__FUNCTION__, __LINE__,
sStart,
_ReadPageEntry(stlbLogical + sStart));
#endif
/* next page. */
start += gcdMMU_PAGE_1M_SIZE;
if (start == 0)
{
physBaseExt++;
}
sStart++;
curStlbChunk->pageCount++;
}
#if gcdDUMP_IN_KERNEL
{
gctUINT32 data = stlbLogical[i] & ~0xF;
gctUINT32 step = (last > i) ? (stlbLogical[i + 1] - stlbLogical[i]) : 0;
gctUINT32 mask = stlbLogical[i] & 0xF;
gcmkDUMP(Mmu->os,
"@[physical.step 0x%010llX 0x%08X 0x%08X 0x%08X 0x%08X]",
(unsigned long long)stlbPhyBase + i * 4,
data, (last - i) * 4, step, mask);
}
#endif
gcmkONERROR(gckVIDMEM_NODE_CleanCache(
kernel,
curStlbChunk->videoMem,
0,
curStlbChunk->logical,
curStlbChunk->size
));
sStart = 0;
++mStart;
}
gcmkASSERT(totalNewStlbs == stlbIndex);
if (newStlbChunk)
{
/* Insert the stlbChunk into staticSTLB. */
if (Mmu->staticSTLB == gcvNULL)
{
Mmu->staticSTLB = newStlbChunk;
}
else
{
gcmkASSERT(newStlbChunk != gcvNULL);
gcmkASSERT(newStlbChunk->next == gcvNULL);
newStlbChunk->next = Mmu->staticSTLB;
Mmu->staticSTLB = newStlbChunk;
}
}
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
#if gcdENABLE_TRUST_APPLICATION
if (Mmu->hardware->options.secureMode == gcvSECURE_IN_TA)
{
gckKERNEL_SecurityMapMemory(Mmu->hardware->kernel, gcvNULL, physBase, flatSize / gcdMMU_PAGE_4K_SIZE, &physBase);
}
#endif
/* Store the gpu physical ranges */
Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].start = flatBase;
Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].end = flatBase + flatSize - 1;
Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].size = flatSize;
Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].flag = mapFlag;
Mmu->gpuPhysicalRangeCount++;
gcmkASSERT(Mmu->gpuPhysicalRangeCount <= gcdMAX_FLAT_MAPPING_COUNT);
/* Store the gpu virtual ranges */
Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].start = flatVirtualBase;
Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].end = flatVirtualBase + flatSize - 1;
Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].size = flatSize;
Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].flag = mapFlag;
Mmu->gpuAddressRangeCount++;
gcmkASSERT(Mmu->gpuAddressRangeCount <= gcdMAX_FLAT_MAPPING_COUNT);
return gcvSTATUS_OK;
OnError:
/* Roll back the allocation.
** We don't need roll back mtlb programming as gckmONERROR
** is only used during allocation time.
*/
if (newStlbChunk)
{
if (newStlbChunk->videoMem)
{
gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(
kernel,
newStlbChunk->videoMem
));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, newStlbChunk));
}
if (mutex)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
}
return status;
}
}
static gceSTATUS
_SetupAddressArea(
IN gckOS Os,
IN gcsADDRESS_AREA_PTR Area,
IN gctUINT32 NumMTLBEntries
)
{
gceSTATUS status;
gctUINT32_PTR map;
gctUINT32 stlbSize = (Area->areaType == gcvAREA_TYPE_1M)
? gcdMMU_STLB_1M_SIZE : gcdMMU_STLB_4K_SIZE;
gcmkHEADER();
Area->stlbSize = NumMTLBEntries * stlbSize;
gcmkSAFECASTSIZET(Area->stlbEntries, Area->stlbSize / gcmSIZEOF(gctUINT32));
gcmkONERROR(gckOS_Allocate(Os, Area->stlbSize, (void **)&Area->mapLogical));
/* Initialization. */
map = Area->mapLogical;
map[0] = (Area->stlbEntries << 8) | gcvMMU_FREE;
map[1] = ~0U;
Area->heapList = 0;
Area->freeNodes = gcvFALSE;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
static gceSTATUS
_ConstructDynamicStlb(
IN gckMMU Mmu,
IN gcsADDRESS_AREA_PTR Area,
IN gctUINT32 NumEntries
)
{
gceSTATUS status = gcvSTATUS_OK;
gctBOOL acquired = gcvFALSE;
gckKERNEL kernel = Mmu->hardware->kernel;
gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS;
gcePOOL pool = Mmu->pool;
gctUINT32 address;
gctUINT32 mtlbEntry;
gctUINT32 i;
#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
allocFlag |= gcvALLOC_FLAG_CACHEABLE;
#endif
allocFlag |= gcvALLOC_FLAG_4GB_ADDR;
/* Construct Slave TLB. */
gcmkONERROR(gckKERNEL_AllocateVideoMemory(
kernel,
64,
gcvVIDMEM_TYPE_COMMAND,
allocFlag | gcvALLOC_FLAG_4K_PAGES,
&Area->stlbSize,
&pool,
&Area->stlbVideoMem));
/* Lock for kernel side CPU access. */
gcmkONERROR(gckVIDMEM_NODE_LockCPU(
kernel,
Area->stlbVideoMem,
gcvFALSE,
gcvFALSE,
(gctPOINTER *)&Area->stlbLogical));
#if gcdUSE_MMU_EXCEPTION
gcmkONERROR(_FillPageTable(Area->stlbLogical,
Area->stlbEntries,
/* Enable exception */
1 << 1));
#else
/* Invalidate all entries. */
gcmkONERROR(gckOS_ZeroMemory(Area->stlbLogical,
Area->stlbSize));
#endif
/* Get stlb table physical. */
gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
kernel,
Area->stlbVideoMem,
0,
&Area->stlbPhysical));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Mmu->os,
Area->stlbPhysical,
&Area->stlbPhysical));
if (Area->areaType == gcvAREA_TYPE_1M)
{
gcmkDUMP(Mmu->os, "#[mmu: 1M page size dynamic space: 0x%08X - 0x%08X]",
(Area->mappingStart << gcdMMU_MTLB_SHIFT),
(Area->mappingEnd << gcdMMU_MTLB_SHIFT) - 1);
}
else
{
gcmkDUMP(Mmu->os, "#[mmu: 4K page size dynamic space: 0x%08X - 0x%08X]",
(Area->mappingStart << gcdMMU_MTLB_SHIFT),
(Area->mappingEnd << gcdMMU_MTLB_SHIFT) - 1);
}
gcmkDUMP(Mmu->os, "#[mmu-stlb]");
gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08lX]",
(unsigned long long)Area->stlbPhysical,
Area->stlbLogical[0],
(unsigned long)Area->stlbSize);
gcmkSAFECASTPHYSADDRT(address, Area->stlbPhysical);
/* Grab the mutex. */
gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
acquired = gcvTRUE;
/* Map to Master TLB. */
for (i = Area->mappingStart;
i < Area->mappingStart + NumEntries;
i++)
{
if (Area->areaType == gcvAREA_TYPE_1M)
{
mtlbEntry = address
/* 1M page size */
| (1 << 3)
/*Ignore exception */
| (0 << 1)
/* Present */
| (1 << 0);
address += gcdMMU_STLB_1M_SIZE;
}
else
{
mtlbEntry = address
/* 4KB page size */
| (0 << 2)
/*Ignore exception */
| (0 << 1)
/* Present */
| (1 << 0);
address += gcdMMU_STLB_4K_SIZE;
}
_WritePageEntry(Mmu->mtlbLogical + i, mtlbEntry);
#if gcdMMU_TABLE_DUMP
gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n",
__FUNCTION__, __LINE__,
i,
_ReadPageEntry(Mmu->mtlbLogical + i));
#endif
}
gcmkDUMP(Mmu->os, "#[mmu-mtlb: slot: %d - %d]",
Area->mappingStart, Area->mappingEnd - 1);
#if gcdDUMP_IN_KERNEL
{
gctUINT32 data = Mmu->mtlbLogical[Area->mappingStart] & ~0x3F;
gctUINT32 step = 0;
gctUINT32 mask = Mmu->mtlbLogical[Area->mappingStart] & 0x3F;
if (NumEntries > 1)
{
step = Mmu->mtlbLogical[Area->mappingStart + 1]
- Mmu->mtlbLogical[Area->mappingStart];
}
gcmkDUMP(Mmu->os,
"@[physical.step 0x%010llX 0x%08X 0x%08X 0x%08X 0x%08X]",
(unsigned long long)(Mmu->mtlbPhysical + Area->mappingStart * 4),
data, NumEntries * 4, step, mask);
}
#endif
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
}
return status;
}
gceSTATUS
gckMMU_SetupDynamicSpace(
IN gckMMU Mmu
)
{
gceSTATUS status;
gcsFreeSpaceNode_PTR nodeArray = gcvNULL;
gctINT i, nodeArraySize = 0;
gctINT numEntries = 0;
gckKERNEL kernel = Mmu->hardware->kernel;
gcsADDRESS_AREA_PTR area4K = &Mmu->dynamicArea4K;
#if gcdENABLE_GPU_1M_PAGE
gcsADDRESS_AREA_PTR area1M = &Mmu->dynamicArea1M;
gctINT numEntries1M;
#endif
/* Find all the free address space. */
gcmkONERROR(_CollectFreeSpace(Mmu, &nodeArray, &nodeArraySize));
for (i = 0; i < nodeArraySize; i++)
{
if (nodeArray[i].entries > numEntries)
{
area4K->mappingStart = nodeArray[i].start;
numEntries = nodeArray[i].entries;
area4K->mappingEnd = area4K->mappingStart + numEntries - 1;
}
}
gckOS_Free(Mmu->os, (gctPOINTER)nodeArray);
#if gcdENABLE_TRUST_APPLICATION
if (gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_SECURITY) == gcvSTATUS_TRUE)
{
/* Setup secure address area when needed. */
gctUINT32 secureAreaSize = gcdMMU_SECURE_AREA_SIZE;
gcsADDRESS_AREA_PTR secureArea = &Mmu->secureArea;
gcmkASSERT(numEntries > (gctINT)secureAreaSize);
secureArea->mappingStart = area4K->mappingStart
+ (numEntries - secureAreaSize);
gcmkONERROR(_SetupAddressArea(Mmu->os, secureArea, secureAreaSize));
numEntries -= secureAreaSize;
area4K->mappingEnd -= secureAreaSize;
}
#endif
#if gcdENABLE_GPU_1M_PAGE
numEntries1M = numEntries >> 1;
area1M->mappingStart = area4K->mappingStart + (numEntries - numEntries1M);
area1M->mappingEnd = area1M->mappingStart + numEntries1M - 1;
area1M->areaType = gcvAREA_TYPE_1M;
numEntries -= numEntries1M;
area4K->mappingEnd -= numEntries1M;
gcmkONERROR(_SetupAddressArea(Mmu->os, area1M, numEntries1M));
gcmkONERROR(_ConstructDynamicStlb(Mmu, area1M, numEntries1M));
#endif
area4K->areaType = gcvAREA_TYPE_4K;
/* Setup normal address area. */
gcmkONERROR(_SetupAddressArea(Mmu->os, area4K, numEntries));
gcmkONERROR(_ConstructDynamicStlb(Mmu, area4K, numEntries));
return gcvSTATUS_OK;
OnError:
#if gcdENABLE_GPU_1M_PAGE
if (area1M->mapLogical)
{
gcmkVERIFY_OK(
gckOS_Free(Mmu->os, (gctPOINTER) area1M->mapLogical));
}
if (area1M->stlbVideoMem)
{
gcmkVERIFY_OK(
gckVIDMEM_NODE_Dereference(kernel,
area1M->stlbVideoMem));
}
#endif
if (area4K->mapLogical)
{
gcmkVERIFY_OK(
gckOS_Free(Mmu->os, (gctPOINTER) area4K->mapLogical));
}
if (area4K->stlbVideoMem)
{
gcmkVERIFY_OK(
gckVIDMEM_NODE_Dereference(kernel,
area4K->stlbVideoMem));
}
return status;
}
gctUINT32
_GetPageCountOfUsedNode(
gctUINT32_PTR Node
)
{
gctUINT32 count;
count = gcmENTRY_COUNT(*Node);
if ((count << 8) == (~((1U<<8)-1)))
{
count = 1;
}
return count;
}
static gcsADDRESS_AREA_PTR
_GetProcessArea(
IN gckMMU Mmu,
IN gcePAGE_TYPE PageType,
IN gctBOOL Secure
)
{
#if gcdENABLE_TRUST_APPLICATION
if (Secure == gcvTRUE)
{
return &Mmu->secureArea;
}
#endif
if (PageType == gcvPAGE_TYPE_1M)
{
return &Mmu->dynamicArea1M;
}
else
{
return &Mmu->dynamicArea4K;
}
}
/*******************************************************************************
**
** _Construct
**
** Construct a new gckMMU object.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctSIZE_T MmuSize
** Number of bytes for the page table.
**
** OUTPUT:
**
** gckMMU * Mmu
** Pointer to a variable that receives the gckMMU object pointer.
*/
gceSTATUS
_Construct(
IN gckKERNEL Kernel,
IN gctSIZE_T MmuSize,
OUT gckMMU * Mmu
)
{
gckOS os;
gckHARDWARE hardware;
gceSTATUS status;
gckMMU mmu = gcvNULL;
gctUINT32_PTR map;
gctPOINTER pointer = gcvNULL;
gctPHYS_ADDR_T physBase;
gctSIZE_T physSize;
gctPHYS_ADDR_T contiguousBase;
gctSIZE_T contiguousSize = 0;
gctPHYS_ADDR_T externalBase;
gctSIZE_T externalSize = 0;
gctUINT32 gpuAddress;
gctPHYS_ADDR_T gpuPhysical;
gcsADDRESS_AREA_PTR area = gcvNULL;
gcePOOL pool;
gctUINT64 data;
gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS;
gctUINT64 mmuEnabled;
gcmkHEADER_ARG("Kernel=0x%x MmuSize=%lu", Kernel, MmuSize);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(MmuSize > 0);
gcmkVERIFY_ARGUMENT(Mmu != gcvNULL);
/* Extract the gckOS object pointer. */
os = Kernel->os;
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
/* Extract the gckHARDWARE object pointer. */
hardware = Kernel->hardware;
gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
/* Allocate memory for the gckMMU object. */
gcmkONERROR(gckOS_Allocate(os, sizeof(struct _gckMMU), &pointer));
gckOS_ZeroMemory(pointer, sizeof(struct _gckMMU));
mmu = pointer;
/* Initialize the gckMMU object. */
mmu->object.type = gcvOBJ_MMU;
mmu->os = os;
mmu->hardware = hardware;
mmu->pageTableMutex = gcvNULL;
mmu->mtlbLogical = gcvNULL;
mmu->staticSTLB = gcvNULL;
mmu->enabled = gcvFALSE;
mmu->dynamicAreaSetuped = gcvFALSE;
mmu->pool = _GetPageTablePool(mmu->os);
gcsLIST_Init(&mmu->hardwareList);
/* Use 4K page size for MMU version 0. */
area = &mmu->dynamicArea4K;
area->mapLogical = gcvNULL;
area->stlbLogical = gcvNULL;
/* Create the page table mutex. */
gcmkONERROR(gckOS_CreateMutex(os, &mmu->pageTableMutex));
gcmkONERROR(gckOS_QueryOption(os, "mmu", &mmuEnabled));
if (hardware->mmuVersion == 0)
{
area->stlbSize = MmuSize;
/* Construct address space management table. */
gcmkONERROR(gckOS_Allocate(mmu->os,
area->stlbSize,
&pointer));
area->mapLogical = pointer;
pool = mmu->pool;
#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
allocFlag |= gcvALLOC_FLAG_CACHEABLE;
#endif
/* Construct page table read by GPU. */
gcmkONERROR(gckKERNEL_AllocateVideoMemory(
Kernel,
4096,
gcvVIDMEM_TYPE_COMMAND,
allocFlag,
&area->stlbSize,
&pool,
&area->stlbVideoMem));
/* Lock for kernel side CPU access. */
gcmkONERROR(gckVIDMEM_NODE_LockCPU(
Kernel,
area->stlbVideoMem,
gcvFALSE,
gcvFALSE,
&pointer));
area->stlbLogical = pointer;
/* Get CPU physical address. */
gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
Kernel,
area->stlbVideoMem,
0,
&area->stlbPhysical));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(mmu->os,
area->stlbPhysical,
&area->stlbPhysical));
/* Compute number of entries in page table. */
gcmkSAFECASTSIZET(area->stlbEntries, area->stlbSize / sizeof(gctUINT32));
/* Mark all pages as free. */
map = area->mapLogical;
_FillPageTable(area->stlbLogical, area->stlbEntries, mmu->safeAddress);
gcmkDUMP(mmu->os,
"#[mmu0: fill with safe address]");
gcmkDUMP(mmu->os,
"@[physical.fill 0x%010llX 0x%08X 0x%08lX]",
(unsigned long long)area->stlbPhysical,
area->stlbLogical[0],
(unsigned long)area->stlbSize);
map[0] = (area->stlbEntries << 8) | gcvMMU_FREE;
map[1] = ~0U;
area->heapList = 0;
area->freeNodes = gcvFALSE;
status = gckOS_QueryOption(mmu->os, "contiguousBase", &contiguousBase);
if (gcmIS_SUCCESS(status))
{
status = gckOS_QueryOption(mmu->os, "contiguousSize", &data);
contiguousSize = (gctSIZE_T)data;
}
if (gcmIS_SUCCESS(status) && contiguousSize)
{
/* Convert to GPU address. */
mmu->contiguousBaseAddress = (gctUINT32)(contiguousBase - Kernel->hardware->baseAddress);
}
}
else
{
mmu->mtlbSize = gcdMMU_MTLB_SIZE;
pool = mmu->pool;
#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
allocFlag |= gcvALLOC_FLAG_CACHEABLE;
#endif
allocFlag |= gcvALLOC_FLAG_4GB_ADDR;
/* 1K mode is 1024 byte aligned. */
gcmkONERROR(gckKERNEL_AllocateVideoMemory(
Kernel,
1024,
gcvVIDMEM_TYPE_COMMAND,
allocFlag | gcvALLOC_FLAG_4K_PAGES,
&mmu->mtlbSize,
&pool,
&mmu->mtlbVideoMem));
/* Lock for kernel side CPU access. */
gcmkONERROR(gckVIDMEM_NODE_LockCPU(
Kernel,
mmu->mtlbVideoMem,
gcvFALSE,
gcvFALSE,
&pointer));
mmu->mtlbLogical = pointer;
mmu->dynamicArea4K.mappingStart = gcvINVALID_ADDRESS;
mmu->dynamicArea1M.mappingStart = gcvINVALID_ADDRESS;
/* Get mtlb table physical. */
gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
Kernel,
mmu->mtlbVideoMem,
0,
&mmu->mtlbPhysical));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(
mmu->os,
mmu->mtlbPhysical,
&mmu->mtlbPhysical));
/* Invalid all the entries. */
gcmkONERROR(
gckOS_ZeroMemory(pointer, mmu->mtlbSize));
gcmkONERROR(
gckOS_QueryOption(mmu->os, "physBase", &physBase));
gcmkONERROR(
gckOS_QueryOption(mmu->os, "physSize", &data));
physSize = (gctSIZE_T)data;
gcmkONERROR(
gckOS_CPUPhysicalToGPUPhysical(mmu->os, physBase, &gpuPhysical));
gcmkSAFECASTPHYSADDRT(gpuAddress, gpuPhysical);
if (physSize)
{
/* Setup user specified flat mapping. */
gcmkONERROR(_FillFlatMapping(mmu, gpuAddress, physSize, gcvFALSE, gcvFALSE, gcvNULL));
}
#if !(0 || gcdCAPTURE_ONLY_MODE)
if (!_ReadPageEntry(mmu->mtlbLogical + 0))
{
gctUINT32 mtlbEntry;
/*
* Reserved the first mtlb.
* 1MB page size, Ingore exception, Not Present.
*/
mtlbEntry = (1 << 3)
| (0 << 1)
| (0 << 0);
_WritePageEntry(mmu->mtlbLogical + 0, mtlbEntry);
gcmkDUMP(mmu->os, "#[mmu-mtlb: reserved 16M space, slot: 0]");
gcmkDUMP(mmu->os,
"@[physical.fill 0x%010llX 0x%08X 0x%08X]",
(unsigned long long)mmu->mtlbPhysical,
mmu->mtlbLogical[0], 4);
/* Store the gpu virtual ranges */
mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].start = 0;
mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].end = (16 << 20) - 1;
mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].size = (16 << 20);
mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].flag = gcvFLATMAP_DIRECT;
mmu->gpuAddressRangeCount++;
}
#endif
status = gckOS_QueryOption(mmu->os, "contiguousBase", &contiguousBase);
if (gcmIS_SUCCESS(status))
{
status = gckOS_QueryOption(mmu->os, "contiguousSize", &data);
contiguousSize = (gctSIZE_T)data;
}
if (gcmIS_SUCCESS(status) && contiguousSize)
{
gctUINT64 gpuContiguousBase;
gctUINT32 contiguousBaseAddress = 0;
gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(mmu->os, contiguousBase, &gpuContiguousBase));
/* Setup flat mapping for reserved memory (VIDMEM). */
gcmkONERROR(_FillFlatMapping(mmu, gpuContiguousBase, contiguousSize, gcvFALSE, gcvTRUE, &contiguousBaseAddress));
if (mmuEnabled)
{
mmu->contiguousBaseAddress = contiguousBaseAddress;
}
else
{
gcmkSAFECASTPHYSADDRT(mmu->contiguousBaseAddress, gpuContiguousBase);
}
}
status = gckOS_QueryOption(mmu->os, "externalBase", &externalBase);
if (gcmIS_SUCCESS(status))
{
status = gckOS_QueryOption(mmu->os, "externalSize", &data);
externalSize = (gctSIZE_T)data;
}
if (gcmIS_SUCCESS(status) && externalSize)
{
gctUINT64 gpuExternalBase;
gctUINT32 externalBaseAddress = 0;
gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(mmu->os, externalBase, &gpuExternalBase));
/* Setup flat mapping for external memory. */
gcmkONERROR(_FillFlatMapping(mmu, gpuExternalBase, externalSize, gcvFALSE, gcvTRUE, &externalBaseAddress));
mmu->externalBaseAddress = externalBaseAddress;
}
}
/* A 64 byte for safe address, we use 256 here. */
mmu->safePageSize = 256;
pool = mmu->pool;
/* Allocate safe page from video memory. */
gcmkONERROR(gckKERNEL_AllocateVideoMemory(
Kernel,
256,
gcvVIDMEM_TYPE_COMMAND,
gcvALLOC_FLAG_CONTIGUOUS | gcvALLOC_FLAG_4K_PAGES | gcvALLOC_FLAG_4GB_ADDR,
&mmu->safePageSize,
&pool,
&mmu->safePageVideoMem
));
/* Lock for kernel side CPU access. */
gcmkONERROR(gckVIDMEM_NODE_LockCPU(
Kernel,
mmu->safePageVideoMem,
gcvFALSE,
gcvFALSE,
&mmu->safePageLogical
));
/* Get CPU physical address. */
gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
Kernel,
mmu->safePageVideoMem,
0,
&mmu->safePagePhysical
));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(
os,
mmu->safePagePhysical,
&mmu->safePagePhysical
));
gcmkSAFECASTPHYSADDRT(mmu->safeAddress, mmu->safePagePhysical);
gckOS_ZeroMemory(mmu->safePageLogical, mmu->safePageSize);
gcmkDUMP(mmu->os, "#[safe page]");
gcmkDUMP(mmu->os,
"@[physical.fill 0x%010llX 0x%08X 0x%08lX]",
(unsigned long long)mmu->safePagePhysical,
0,
(unsigned long)mmu->safePageSize);
gcmkDUMP_BUFFER(
mmu->os,
gcvDUMP_BUFFER_KERNEL_COMMAND,
mmu->safePageLogical,
mmu->safeAddress,
mmu->safePageSize
);
gcmkONERROR(gckQUEUE_Allocate(os, &mmu->recentFreedAddresses, 16));
mmu->sRAMMapped = gcvFALSE;
/* Return the gckMMU object pointer. */
*Mmu = mmu;
/* Success. */
gcmkFOOTER_ARG("*Mmu=0x%x", *Mmu);
return gcvSTATUS_OK;
OnError:
/* Roll back. */
if (mmu != gcvNULL)
{
if (area != gcvNULL && area->mapLogical != gcvNULL)
{
gcmkVERIFY_OK(
gckOS_Free(os, (gctPOINTER) area->mapLogical));
gcmkVERIFY_OK(
gckVIDMEM_NODE_Dereference(Kernel,
area->stlbVideoMem));
}
if (mmu->mtlbLogical != gcvNULL)
{
gcmkVERIFY_OK(
gckVIDMEM_NODE_Dereference(Kernel,
mmu->mtlbVideoMem));
}
if (mmu->pageTableMutex != gcvNULL)
{
/* Delete the mutex. */
gcmkVERIFY_OK(
gckOS_DeleteMutex(os, mmu->pageTableMutex));
}
gcmkVERIFY_OK(gckQUEUE_Free(os, &mmu->recentFreedAddresses));
/* Mark the gckMMU object as unknown. */
mmu->object.type = gcvOBJ_UNKNOWN;
/* Free the allocates memory. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, mmu));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
static gceSTATUS
_FreeAddressArea(
gckKERNEL Kernel,
gcsADDRESS_AREA * Area
)
{
gceSTATUS status = gcvSTATUS_OK;
if (Area->mapLogical != gcvNULL)
{
gcmkVERIFY_OK(
gckOS_Free(Kernel->os, (gctPOINTER) Area->mapLogical));
}
if (Area->stlbLogical != gcvNULL)
{
/* Free page table. */
gcmkVERIFY_OK(
gckVIDMEM_NODE_Dereference(Kernel,
Area->stlbVideoMem));
}
return status;
}
/*******************************************************************************
**
** _Destroy
**
** Destroy a gckMMU object.
**
** INPUT:
**
** gckMMU Mmu
** Pointer to an gckMMU object.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
_Destroy(
IN gckMMU Mmu
)
{
gckKERNEL kernel = Mmu->hardware->kernel;
gcmkHEADER_ARG("Mmu=0x%x", Mmu);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
while (Mmu->staticSTLB != gcvNULL)
{
gcsMMU_STLB_CHUNK_PTR pre = Mmu->staticSTLB;
Mmu->staticSTLB = pre->next;
if (pre->videoMem)
{
gcmkVERIFY_OK(
gckVIDMEM_NODE_Dereference(kernel,
pre->videoMem));
}
if (pre->mtlbEntryNum != 0)
{
gctUINT i;
for (i = 0; i < pre->mtlbEntryNum; ++i)
{
_WritePageEntry(Mmu->mtlbLogical + pre->mtlbIndex + i, 0);
#if gcdMMU_TABLE_DUMP
gckOS_Print("%s(%d): clean MTLB[%d]\n",
__FUNCTION__, __LINE__,
pre->mtlbIndex + i);
#endif
}
gcmkDUMP(Mmu->os,
"#[mmu-mtlb: clean up slot: %d - %d]",
pre->mtlbIndex,
pre->mtlbIndex + pre->mtlbEntryNum - 1);
gcmkDUMP(Mmu->os,
"@[physical.fill 0x%010llX 0x%08X 0x%08lX]",
(unsigned long long)(Mmu->mtlbPhysical + pre->mtlbIndex * 4),
Mmu->mtlbLogical[pre->mtlbIndex],
(unsigned long)(pre->mtlbEntryNum * 4));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, pre));
}
if (Mmu->hardware->mmuVersion != 0)
{
gcmkVERIFY_OK(
gckVIDMEM_NODE_Dereference(kernel,
Mmu->mtlbVideoMem));
}
/* Free address area. */
gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->dynamicArea4K));
#if gcdENABLE_GPU_1M_PAGE
gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->dynamicArea1M));
#endif
gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->secureArea));
/* Delete the page table mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Mmu->os, Mmu->pageTableMutex));
if (Mmu->safePageLogical != gcvNULL)
{
gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(
kernel,
Mmu->safePageVideoMem
));
}
gcmkVERIFY_OK(gckQUEUE_Free(Mmu->os, &Mmu->recentFreedAddresses));
/* Mark the gckMMU object as unknown. */
Mmu->object.type = gcvOBJ_UNKNOWN;
/* Free the gckMMU object. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, Mmu));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
** _AdjstIndex
**
** Adjust the index from which we search for a usable node to make sure
** index allocated is greater than Start.
*/
gceSTATUS
_AdjustIndex(
IN gckMMU Mmu,
IN gctUINT32 Index,
IN gctUINT32 PageCount,
IN gctUINT32 Start,
OUT gctUINT32 * IndexAdjusted
)
{
gceSTATUS status;
gctUINT32 index = Index;
gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K;
gctUINT32_PTR map = area->mapLogical;
gcmkHEADER();
for (; index < area->stlbEntries;)
{
gctUINT32 result = 0;
gctUINT32 nodeSize = 0;
if (index >= Start)
{
break;
}
switch (gcmENTRY_TYPE(map[index]))
{
case gcvMMU_SINGLE:
nodeSize = 1;
break;
case gcvMMU_FREE:
nodeSize = map[index] >> 8;
break;
default:
gcmkFATAL("MMU table correcupted at index %u!", index);
gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
}
if (nodeSize > PageCount)
{
result = index + (nodeSize - PageCount);
if (result >= Start)
{
break;
}
}
switch (gcmENTRY_TYPE(map[index]))
{
case gcvMMU_SINGLE:
index = map[index] >> 8;
break;
case gcvMMU_FREE:
index = map[index + 1];
break;
default:
gcmkFATAL("MMU table correcupted at index %u!", index);
gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
}
}
*IndexAdjusted = index;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
gceSTATUS
gckMMU_Construct(
IN gckKERNEL Kernel,
IN gctSIZE_T MmuSize,
OUT gckMMU * Mmu
)
{
#if gcdSHARED_PAGETABLE
gceSTATUS status;
gctPOINTER pointer;
gcmkHEADER_ARG("Kernel=0x%08x", Kernel);
if (sharedPageTable == gcvNULL)
{
gcmkONERROR(
gckOS_Allocate(Kernel->os,
sizeof(struct _gcsSharedPageTable),
&pointer));
sharedPageTable = pointer;
gcmkONERROR(
gckOS_ZeroMemory(sharedPageTable,
sizeof(struct _gcsSharedPageTable)));
gcmkONERROR(_Construct(Kernel, MmuSize, &sharedPageTable->mmu));
}
*Mmu = sharedPageTable->mmu;
sharedPageTable->hardwares[sharedPageTable->reference] = Kernel->hardware;
sharedPageTable->reference++;
gcmkFOOTER_ARG("sharedPageTable->reference=%lu", sharedPageTable->reference);
return gcvSTATUS_OK;
OnError:
if (sharedPageTable)
{
if (sharedPageTable->mmu)
{
gcmkVERIFY_OK(gckMMU_Destroy(sharedPageTable->mmu));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, sharedPageTable));
}
gcmkFOOTER();
return status;
#else
return _Construct(Kernel, MmuSize, Mmu);
#endif
}
gceSTATUS
gckMMU_Destroy(
IN gckMMU Mmu
)
{
#if gcdSHARED_PAGETABLE
gckOS os = Mmu->os;
sharedPageTable->reference--;
if (sharedPageTable->reference == 0)
{
if (sharedPageTable->mmu)
{
gcmkVERIFY_OK(_Destroy(Mmu));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, sharedPageTable));
}
return gcvSTATUS_OK;
#else
return _Destroy(Mmu);
#endif
}
/*******************************************************************************
**
** gckMMU_AllocatePages
**
** Allocate pages inside the page table.
**
** INPUT:
**
** gckMMU Mmu
** Pointer to an gckMMU object.
**
** gctSIZE_T PageCount
** Number of pages to allocate.
**
** OUTPUT:
**
** gctPOINTER * PageTable
** Pointer to a variable that receives the base address of the page
** table.
**
** gctUINT32 * Address
** Pointer to a variable that receives the hardware specific address.
*/
gceSTATUS
_AllocatePages(
IN gckMMU Mmu,
IN gctSIZE_T PageCount,
IN gceVIDMEM_TYPE Type,
IN gcePAGE_TYPE PageType,
IN gctBOOL Secure,
OUT gctPOINTER * PageTable,
OUT gctUINT32 * Address
)
{
gceSTATUS status;
gctBOOL mutex = gcvFALSE;
gctUINT32 index = 0, previous = ~0U, left;
gctUINT32_PTR map;
gctBOOL gotIt;
gctUINT32 address;
gctUINT32 pageCount;
gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, Secure);
gcmkHEADER_ARG("Mmu=0x%x PageCount=%lu", Mmu, PageCount);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
gcmkVERIFY_ARGUMENT(PageCount > 0);
gcmkVERIFY_ARGUMENT(PageTable != gcvNULL);
if (PageCount > area->stlbEntries)
{
/* Not enough pages avaiable. */
gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
}
gcmkSAFECASTSIZET(pageCount, PageCount);
#if gcdBOUNDARY_CHECK
/* Extra pages as bounary. */
pageCount += gcdBOUNDARY_CHECK * 2;
#endif
/* Grab the mutex. */
gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
mutex = gcvTRUE;
/* Cast pointer to page table. */
for (map = area->mapLogical, gotIt = gcvFALSE; !gotIt;)
{
index = area->heapList;
if ((Mmu->hardware->mmuVersion == 0) &&
(Type == gcvVIDMEM_TYPE_VERTEX_BUFFER))
{
gcmkONERROR(_AdjustIndex(
Mmu,
index,
pageCount,
gcdVERTEX_START / gcmSIZEOF(gctUINT32),
&index
));
}
/* Walk the heap list. */
for (; !gotIt && (index < area->stlbEntries);)
{
/* Check the node type. */
switch (gcmENTRY_TYPE(map[index]))
{
case gcvMMU_SINGLE:
/* Single odes are valid if we only need 1 page. */
if (pageCount == 1)
{
gotIt = gcvTRUE;
}
else
{
/* Move to next node. */
previous = index;
index = map[index] >> 8;
}
break;
case gcvMMU_FREE:
/* Test if the node has enough space. */
if (pageCount <= (map[index] >> 8))
{
gotIt = gcvTRUE;
}
else
{
/* Move to next node. */
previous = index;
index = map[index + 1];
}
break;
default:
gcmkFATAL("MMU table correcupted at index %u!", index);
gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
}
}
/* Test if we are out of memory. */
if (index >= area->stlbEntries)
{
if (area->freeNodes)
{
/* Time to move out the trash! */
gcmkONERROR(_Collect(area));
/* We are going to search from start, so reset previous to start. */
previous = ~0U;
}
else
{
/* Out of resources. */
gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
}
}
}
switch (gcmENTRY_TYPE(map[index]))
{
case gcvMMU_SINGLE:
/* Unlink single node from free list. */
gcmkONERROR(
_Link(area, previous, map[index] >> 8));
break;
case gcvMMU_FREE:
/* Check how many pages will be left. */
left = (map[index] >> 8) - pageCount;
switch (left)
{
case 0:
/* The entire node is consumed, just unlink it. */
gcmkONERROR(
_Link(area, previous, map[index + 1]));
break;
case 1:
/* One page will remain. Convert the node to a single node and
** advance the index. */
map[index] = (map[index + 1] << 8) | gcvMMU_SINGLE;
index ++;
break;
default:
/* Enough pages remain for a new node. However, we will just adjust
** the size of the current node and advance the index. */
map[index] = (left << 8) | gcvMMU_FREE;
index += left;
break;
}
break;
}
/* Mark node as used. */
gcmkONERROR(_FillMap(&map[index], pageCount, gcvMMU_USED));
#if gcdBOUNDARY_CHECK
index += gcdBOUNDARY_CHECK;
#endif
/* Record pageCount of allocated node at the beginning of node. */
if (pageCount == 1)
{
map[index] = (~((1U<<8)-1)) | gcvMMU_USED;
}
else
{
map[index] = (pageCount << 8) | gcvMMU_USED;
}
if (area->stlbLogical != gcvNULL)
{
/* Return pointer to page table. */
*PageTable = &area->stlbLogical[index];
}
else
{
/* Page table for secure area is handled in trust application. */
*PageTable = gcvNULL;
}
/* Build virtual address. */
if (Mmu->hardware->mmuVersion == 0)
{
gcmkONERROR(
gckHARDWARE_BuildVirtualAddress(Mmu->hardware, index, 0, &address));
}
else
{
gctUINT32 num = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM;
gctUINT32 shift = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT;
gctUINT32 masterOffset = index / num + area->mappingStart;
gctUINT32 slaveOffset = index % num;
address = (masterOffset << gcdMMU_MTLB_SHIFT)
| (slaveOffset << shift);
}
if (Address != gcvNULL)
{
*Address = address;
}
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
/* Success. */
gcmkFOOTER_ARG("*PageTable=0x%x *Address=%08x",
*PageTable, gcmOPT_VALUE(Address));
return gcvSTATUS_OK;
OnError:
if (mutex)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckMMU_FreePages
**
** Free pages inside the page table.
**
** INPUT:
**
** gckMMU Mmu
** Pointer to an gckMMU object.
**
** gctPOINTER PageTable
** Base address of the page table to free.
**
** gctSIZE_T PageCount
** Number of pages to free.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
_FreePages(
IN gckMMU Mmu,
IN gctBOOL Secure,
IN gcePAGE_TYPE PageType,
IN gctUINT32 Address,
IN gctPOINTER PageTable,
IN gctSIZE_T PageCount
)
{
gctUINT32 index;
gctUINT32_PTR node;
gceSTATUS status;
gctBOOL acquired = gcvFALSE;
gctUINT32 pageCount;
gcuQUEUEDATA data;
gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, gcvFALSE);
gctUINT32 pageSize = (PageType == gcvPAGE_TYPE_1M)
? gcdMMU_PAGE_1M_SIZE : gcdMMU_PAGE_4K_SIZE;
gcmkHEADER_ARG("Mmu=0x%x PageTable=0x%x PageCount=%lu",
Mmu, PageTable, PageCount);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
gcmkVERIFY_ARGUMENT(PageCount > 0);
gcmkSAFECASTSIZET(pageCount, PageCount);
#if gcdBOUNDARY_CHECK
pageCount += gcdBOUNDARY_CHECK * 2;
#endif
/* Get the node by index. */
index = (gctUINT32)((gctUINT32_PTR)PageTable - area->stlbLogical);
node = area->mapLogical + index;
if (pageCount != _GetPageCountOfUsedNode(node))
{
gcmkONERROR(gcvSTATUS_INVALID_REQUEST);
}
#if gcdBOUNDARY_CHECK
node -= gcdBOUNDARY_CHECK;
#endif
gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
acquired = gcvTRUE;
if (Mmu->hardware->mmuVersion == 0)
{
_FillPageTable(PageTable, pageCount, Mmu->safeAddress);
}
if (pageCount == 1)
{
/* Single page node. */
node[0] = (~((1U<<8)-1)) | gcvMMU_SINGLE;
if (PageTable != gcvNULL)
{
#if gcdUSE_MMU_EXCEPTION
/* Enable exception */
_WritePageEntry(PageTable, (1 << 1));
#else
_WritePageEntry(PageTable, 0);
#endif
}
}
else
{
/* Mark the node as free. */
node[0] = (pageCount << 8) | gcvMMU_FREE;
node[1] = ~0U;
if (PageTable != gcvNULL)
{
#if gcdUSE_MMU_EXCEPTION
/* Enable exception */
gcmkVERIFY_OK(_FillPageTable(PageTable, (gctUINT32)PageCount, 1 << 1));
#else
gcmkVERIFY_OK(_FillPageTable(PageTable, (gctUINT32)PageCount, 0));
#endif
}
}
gcmkDUMP(Mmu->os,
"#[mmu-stlb: free 0x%08X - 0x%08X]",
Address, Address + pageCount * pageSize - 1);
gcmkDUMP(Mmu->os,
"@[physical.fill 0x%010llX 0x%08X 0x%08X]",
(unsigned long long)(area->stlbPhysical + index * 4),
*(gctUINT32_PTR)PageTable,
pageCount * 4);
/* We have free nodes. */
area->freeNodes = gcvTRUE;
/* Record freed address range. */
data.addressData.start = Address;
data.addressData.end = Address + (gctUINT32)PageCount * pageSize;
gckQUEUE_Enqueue(&Mmu->recentFreedAddresses, &data);
gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
acquired = gcvFALSE;
#if gcdENABLE_TRUST_APPLICATION
if (Mmu->hardware->options.secureMode == gcvSECURE_IN_TA)
{
gckKERNEL_SecurityUnmapMemory(Mmu->hardware->kernel, Address, (gctUINT32)PageCount);
}
#endif
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckMMU_AllocatePages(
IN gckMMU Mmu,
IN gctSIZE_T PageCount,
IN gcePAGE_TYPE PageType,
OUT gctPOINTER * PageTable,
OUT gctUINT32 * Address
)
{
return gckMMU_AllocatePagesEx(
Mmu, PageCount, gcvVIDMEM_TYPE_GENERIC, PageType, gcvFALSE, PageTable, Address);
}
gceSTATUS
gckMMU_AllocatePagesEx(
IN gckMMU Mmu,
IN gctSIZE_T PageCount,
IN gceVIDMEM_TYPE Type,
IN gcePAGE_TYPE PageType,
IN gctBOOL Secure,
OUT gctPOINTER * PageTable,
OUT gctUINT32 * Address
)
{
#if gcdDISABLE_GPU_VIRTUAL_ADDRESS
gcmkPRINT("GPU virtual address is disabled.");
return gcvSTATUS_NOT_SUPPORTED;
#else
return _AllocatePages(Mmu, PageCount, Type, PageType, Secure, PageTable, Address);
#endif
}
gceSTATUS
gckMMU_FreePages(
IN gckMMU Mmu,
IN gctBOOL Secure,
IN gcePAGE_TYPE PageType,
IN gctUINT32 Address,
IN gctPOINTER PageTable,
IN gctSIZE_T PageCount
)
{
return _FreePages(Mmu, Secure, PageType, Address, PageTable, PageCount);
}
gceSTATUS
gckMMU_SetPage(
IN gckMMU Mmu,
IN gctPHYS_ADDR_T PageAddress,
IN gcePAGE_TYPE PageType,
IN gctBOOL Writable,
IN gctUINT32 *PageEntry
)
{
gctUINT32 addressExt;
gctUINT32 address;
gcmkHEADER_ARG("Mmu=0x%x", Mmu);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
gcmkVERIFY_ARGUMENT(PageEntry != gcvNULL);
if (PageType == gcvPAGE_TYPE_1M)
{
gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFFFF));
}
else
{
gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFF));
}
/* [31:0]. */
address = (gctUINT32)(PageAddress & 0xFFFFFFFF);
/* [39:32]. */
addressExt = (gctUINT32)((PageAddress >> 32) & 0xFF);
if (Mmu->hardware->mmuVersion == 0)
{
_WritePageEntry(PageEntry, address);
}
else
{
_WritePageEntry(PageEntry, _SetPage(address, addressExt, gcvTRUE));
}
#ifdef DUMP_IN_KERNEL
{
gctUINT32 *stlbLogical = (PageType == gcvPAGE_TYPE_1M)
? Mmu->dynamicArea1M.stlbLogical
: Mmu->dynamicArea4K.stlbLogical;
gctPHYS_ADDR_T stlbPhysical = (PageType == gcvPAGE_TYPE_1M)
? Mmu->dynamicArea1M.stlbPhysical
: Mmu->dynamicArea4K.stlbPhysical;
gctPHYS_ADDR_T physical = stlbPhysical + (stlbLogical - PageEntry) * 4;
gckDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 4",
physical, *PageEntry);
}
#endif
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckMMU_Flush(
IN gckMMU Mmu,
IN gceVIDMEM_TYPE Type
)
{
gckHARDWARE hardware;
gctUINT32 mask;
gctINT i;
if (Type == gcvVIDMEM_TYPE_VERTEX_BUFFER ||
Type == gcvVIDMEM_TYPE_INDEX_BUFFER ||
Type == gcvVIDMEM_TYPE_COMMAND)
{
mask = gcvPAGE_TABLE_DIRTY_BIT_FE;
}
else
{
mask = gcvPAGE_TABLE_DIRTY_BIT_OTHER;
}
#if gcdSHARED_PAGETABLE
for (i = 0; i < gcdMAX_GPU_COUNT; i++)
{
gctUINT j;
hardware = sharedPageTable->hardwares[i];
if (hardware)
{
for (j = 0; j < gcvENGINE_GPU_ENGINE_COUNT; j++)
{
gcmkVERIFY_OK(gckOS_AtomSetMask(hardware->pageTableDirty[j], mask));
}
}
}
#else
hardware = Mmu->hardware;
for (i = 0 ; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
{
gcmkVERIFY_OK(
gckOS_AtomSetMask(hardware->pageTableDirty[i], mask));
}
{
gcsLISTHEAD_PTR hardwareHead;
gcmkLIST_FOR_EACH(hardwareHead, &Mmu->hardwareList)
{
hardware = gcmCONTAINEROF(hardwareHead, struct _gckHARDWARE, mmuHead);
if (hardware != Mmu->hardware)
{
for (i = 0 ; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
{
gcmkVERIFY_OK(
gckOS_AtomSetMask(hardware->pageTableDirty[i], mask));
}
}
}
}
#endif
return gcvSTATUS_OK;
}
gceSTATUS
gckMMU_DumpPageTableEntry(
IN gckMMU Mmu,
IN gceAREA_TYPE AreaType,
IN gctUINT32 Address
)
{
gctUINT32_PTR pageTable;
gctUINT32 index;
gctUINT32 mtlb, stlb;
gcsADDRESS_AREA_PTR area = (AreaType == gcvAREA_TYPE_4K)
? &Mmu->dynamicArea4K : &Mmu->dynamicArea1M;
gctUINT32 stlbShift = (AreaType == gcvAREA_TYPE_4K)
? gcdMMU_STLB_4K_SHIFT : gcdMMU_STLB_1M_SHIFT;
gctUINT32 stlbMask = (AreaType == gcvAREA_TYPE_4K)
? gcdMMU_STLB_4K_MASK : gcdMMU_STLB_1M_MASK;
gctUINT32 stlbEntryNum = (AreaType == gcvAREA_TYPE_4K)
? gcdMMU_STLB_4K_ENTRY_NUM : gcdMMU_STLB_1M_ENTRY_NUM;
gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address);
gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
gcmkASSERT(Mmu->hardware->mmuVersion > 0);
mtlb = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT;
if (AreaType != gcvAREA_TYPE_FLATMAP)
{
stlb = (Address & stlbMask) >> stlbShift;
pageTable = area->stlbLogical;
index = (mtlb - area->mappingStart)
* stlbEntryNum
+ stlb;
gcmkPRINT(" Page table entry = 0x%08X", _ReadPageEntry(pageTable + index));
}
else
{
gcsMMU_STLB_CHUNK_PTR stlbChunkObj = Mmu->staticSTLB;
gctUINT32 entry = Mmu->mtlbLogical[mtlb];
stlb = (Address & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;
entry &= 0xFFFFFFF0;
while (stlbChunkObj)
{
gctUINT i;
gctBOOL found = gcvFALSE;
for (i = 0; i < stlbChunkObj->mtlbEntryNum; ++i)
{
gctPHYS_ADDR_T stlbPhysBase = stlbChunkObj->physBase + (i * gcdMMU_STLB_1M_SIZE);
gctUINT32_PTR stlbLogical =
(gctUINT32_PTR)((gctUINT8_PTR)stlbChunkObj->logical + (i * gcdMMU_STLB_1M_SIZE));
if (entry == stlbPhysBase)
{
gcmkPRINT(" Page table entry = 0x%08X", stlbLogical[stlb]);
found = gcvTRUE;
break;
}
}
if (found)
break;
stlbChunkObj = stlbChunkObj->next;
}
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
void
gckMMU_CheckSaftPage(
IN gckMMU Mmu
)
{
gctUINT8_PTR safeLogical = Mmu->safePageLogical;
gctUINT32 offsets[] = {
0,
64,
128,
256,
2560,
4000
};
gctUINT32 i = 0;
while (i < gcmCOUNTOF(offsets))
{
if (safeLogical[offsets[i]] != 0)
{
gcmkPRINT("%s(%d) safe page is over written [%d] = %x",
__FUNCTION__, __LINE__, i, safeLogical[offsets[i]]);
}
}
}
void
gckMMU_DumpAddressSpace(
IN gckMMU Mmu
)
{
gctUINT i;
gctUINT next;
/* TODO: */
gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K;
gctUINT32_PTR map = area->mapLogical;
gctBOOL used = gcvFALSE;
gctUINT32 numPages;
/* Grab the mutex. */
gcmkVERIFY_OK(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
/* Find node which contains index. */
for (i = 0; i < area->stlbEntries; i = next)
{
switch (gcmENTRY_TYPE(map[i]))
{
case gcvMMU_SINGLE:
numPages = 1;
next = i + numPages;
used = gcvFALSE;
break;
case gcvMMU_FREE:
numPages = map[i] >> 8;
next = i + numPages;
used = gcvFALSE;
break;
case gcvMMU_USED:
numPages = 1;
next = i + numPages;
used = gcvTRUE;
break;
default:
gcmkFATAL("MMU table correcupted at index %u!", i);
return;
}
if (!used)
{
gcmkPRINT("Available Range [%d - %d)", i, i + numPages);
}
}
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
}
void
gckMMU_DumpRecentFreedAddress(
IN gckMMU Mmu
)
{
gckQUEUE queue = &Mmu->recentFreedAddresses;
gctUINT32 i;
gcuQUEUEDATA *data;
if (queue->count)
{
gcmkPRINT(" Recent %d freed GPU address ranges:", queue->count);
for (i = 0; i < queue->count; i++)
{
gckQUEUE_GetData(queue, i, &data);
gcmkPRINT(" [%08X - %08X]", data->addressData.start, data->addressData.end);
}
}
}
gceSTATUS
gckMMU_FillFlatMapping(
IN gckMMU Mmu,
IN gctUINT64 PhysBase,
IN gctSIZE_T Size
)
{
gceSTATUS status;
gckHARDWARE hardware = Mmu->hardware;
if (hardware->mmuVersion)
{
gcmkONERROR(_FillFlatMapping(Mmu, PhysBase, Size, gcvFALSE, gcvTRUE, gcvNULL));
}
return gcvSTATUS_OK;
OnError:
return status;
}
gceSTATUS
gckMMU_IsFlatMapped(
IN gckMMU Mmu,
IN gctUINT64 Physical,
IN gctUINT32 Address,
OUT gctBOOL *In
)
{
gceSTATUS status;
gctUINT32 i;
gctBOOL inFlatmapping = gcvFALSE;
gcmkHEADER();
gcmkVERIFY_ARGUMENT(In != gcvNULL);
if (gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_MMU) == gcvFALSE)
{
gcmkONERROR(gcvSTATUS_NOT_SUPPORTED);
}
if (Physical != gcvINVALID_PHYSICAL_ADDRESS)
{
for (i = 0; i < Mmu->gpuPhysicalRangeCount; i++)
{
if ((Physical >= Mmu->gpuPhysicalRanges[i].start) &&
(Physical <= Mmu->gpuPhysicalRanges[i].end))
{
inFlatmapping = gcvTRUE;
break;
}
}
}
if (Address != gcvINVALID_ADDRESS)
{
for (i = 0; i < Mmu->gpuAddressRangeCount; i++)
{
if ((Address >= Mmu->gpuAddressRanges[i].start) &&
(Address <= Mmu->gpuAddressRanges[i].end))
{
inFlatmapping = gcvTRUE;
break;
}
}
}
*In = inFlatmapping;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
gceSTATUS
gckMMU_SetupSRAM(
IN gckMMU Mmu,
IN gckHARDWARE Hardware,
IN gckDEVICE Device
)
{
gctBOOL needMapInternalSRAM = gcvFALSE;
gctPHYS_ADDR_T reservedBase = gcvINVALID_PHYSICAL_ADDRESS;
gctUINT32 reservedSize = 0;
gctUINT i = 0;
gctUINT j = 0;
gceSTATUS status;
gckKERNEL kernel = Hardware->kernel;
gcmkHEADER_ARG("Mmu=0x%x Hardware=0x%x", Mmu, Hardware);
gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE);
if (Hardware->mmuVersion == 0)
{
gcmkONERROR(gcvSTATUS_OK);
}
if (!Mmu->sRAMMapped)
{
gctUINT32 address = gcvINVALID_ADDRESS;
gctUINT32 size = 0;
/* Map all the SRAMs in MMU table. */
for (i = 0; i < gcvCORE_COUNT; i++)
{
for (j = gcvSRAM_INTERNAL0; j < gcvSRAM_INTER_COUNT; j++)
{
reservedBase = Device->sRAMBases[i][j];
reservedSize = Device->sRAMSizes[i][j];
Device->sRAMBaseAddresses[i][j] = 0;
needMapInternalSRAM = reservedSize && (reservedBase != gcvINVALID_PHYSICAL_ADDRESS);
/* Map the internal SRAM. */
if (needMapInternalSRAM)
{
if (Device->showSRAMMapInfo)
{
gcmkPRINT("Galcore Info: MMU mapped core%d SRAM base=0x%llx size=0x%x",
i,
reservedBase,
reservedSize
);
}
/*
* Default gpu virtual base = 0.
* It can be specified if not conflict with existing mapping.
*/
gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(
Mmu->os,
reservedBase,
&reservedBase
));
gcmkONERROR(_FillFlatMapping(
Mmu,
reservedBase,
reservedSize,
gcvTRUE,
gcvTRUE,
&Device->sRAMBaseAddresses[i][j]
));
Device->sRAMBases[i][j] = reservedBase;
}
else if (reservedSize && reservedBase == gcvINVALID_PHYSICAL_ADDRESS)
{
/*
* Reserve the internal SRAM range in first reserved MMU mtlb,
* when CPU physical base address is not specified, which means it is reserve usage.
*/
Device->sRAMBaseAddresses[i][j] = (address == gcvINVALID_ADDRESS) ? gcdRESERVE_START :
address + gcmALIGN(size, gcdRESERVE_ALIGN);
Device->sRAMBases[i][j] = address = Device->sRAMBaseAddresses[i][j];
size = Device->sRAMSizes[i][j];
Device->sRAMPhysFaked[i][j] = gcvTRUE;
}
#if gcdCAPTURE_ONLY_MODE
Device->sRAMPhysFaked[i][j] = gcvTRUE;
#endif
}
}
/* Map all the external SRAMs in MMU table. */
for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++)
{
if (Device->extSRAMSizes[i] &&
(Device->extSRAMBases[i] != gcvINVALID_PHYSICAL_ADDRESS))
{
Device->extSRAMBaseAddresses[i] = 0;
gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(
Mmu->os,
Device->extSRAMBases[i],
&Device->extSRAMGPUBases[i]
));
gcmkONERROR(_FillFlatMapping(
Mmu,
Device->extSRAMGPUBases[i],
Device->extSRAMSizes[i],
gcvFALSE,
gcvTRUE,
&Device->extSRAMBaseAddresses[i]
));
Device->extSRAMGPUPhysNames[i] = gckKERNEL_AllocateNameFromPointer(kernel, Device->extSRAMPhysical[i]);
}
}
Mmu->sRAMMapped = gcvTRUE;
}
/* Get per core SRAM hardware base address. */
for (i = gcvSRAM_INTERNAL0; i < gcvSRAM_INTER_COUNT; i++)
{
if (Device->sRAMSizes[Hardware->core][i] &&
(Device->sRAMBaseAddresses[Hardware->core][i]))
{
kernel->sRAMBaseAddresses[i] = Device->sRAMBaseAddresses[Hardware->core][i];
kernel->sRAMSizes[i] = Hardware->options.sRAMSizes[i]
= Device->sRAMSizes[Hardware->core][i];
kernel->sRAMPhysFaked[i] = Device->sRAMPhysFaked[Hardware->core][i];
Hardware->options.sRAMGPUVirtAddrs[i] = Device->sRAMBaseAddresses[Hardware->core][i];
/* If the internal SRAM usage is reserve. */
if (kernel->sRAMPhysFaked[i])
{
/* Use virtual address as the faked physical address which will never be accessed. */
status = gckVIDMEM_Construct(
Mmu->os,
(gctPHYS_ADDR_T)kernel->sRAMBaseAddresses[i],
kernel->sRAMSizes[i],
64,
0,
&kernel->sRAMVidMem[i]
);
if (gcmIS_ERROR(status))
{
kernel->sRAMSizes[i] = 0;
kernel->sRAMVidMem[i] = gcvNULL;
}
else
{
gcmkONERROR(gckOS_RequestReservedMemory(
Mmu->os,
(gctPHYS_ADDR_T)kernel->sRAMBaseAddresses[i],
kernel->sRAMSizes[i],
"Per core SRAM reserve usage region",
gcvTRUE,
&kernel->sRAMPhysical[i]
));
kernel->sRAMVidMem[i]->physical = kernel->sRAMPhysical[i];
}
}
if (Device->showSRAMMapInfo)
{
gcmkPRINT("Galcore Info: MMU mapped core %d SRAM[%d] hardware virtual address=0x%x size=0x%x",
Hardware->core,
i,
kernel->sRAMBaseAddresses[i],
kernel->sRAMSizes[i]
);
}
}
}
for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++)
{
if (Device->extSRAMSizes[i] &&
(Device->extSRAMBases[i] != gcvINVALID_PHYSICAL_ADDRESS))
{
kernel->extSRAMBaseAddresses[i] = Device->extSRAMBaseAddresses[i];
Hardware->options.extSRAMGPUVirtAddrs[i] = Device->extSRAMBaseAddresses[i];
Hardware->options.extSRAMSizes[i] = Device->extSRAMSizes[i];
Hardware->options.extSRAMCPUPhysAddrs[i] = Device->extSRAMBases[i];
Hardware->options.extSRAMGPUPhysAddrs[i] = Device->extSRAMGPUBases[i];
Hardware->options.extSRAMGPUPhysNames[i] = Device->extSRAMGPUPhysNames[i];
if (Device->showSRAMMapInfo)
{
gcmkPRINT("Galcore Info: MMU mapped external shared SRAM[%d] CPU view base=0x%llx GPU view base=0x%llx GPU virtual address=0x%x size=0x%x",
i,
Device->extSRAMBases[i],
Device->extSRAMGPUBases[i],
kernel->extSRAMBaseAddresses[i],
Device->extSRAMSizes[i]
);
}
}
}
gcsLIST_Add(&Hardware->mmuHead, &Mmu->hardwareList);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
/* Return the error. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckMMU_GetPageEntry(
IN gckMMU Mmu,
IN gcePAGE_TYPE PageType,
IN gctUINT32 Address,
IN gctUINT32_PTR *PageTable
)
{
gctUINT32_PTR pageTable;
gctUINT32 index;
gctUINT32 mtlbOffset, stlbOffset;
gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, gcvFALSE);
gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address);
gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
gcmkASSERT(Mmu->hardware->mmuVersion > 0);
mtlbOffset = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT;
if (mtlbOffset >= area->mappingStart)
{
gctUINT32 stlbShift = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT;
gctUINT32 stlbMask = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_MASK : gcdMMU_STLB_4K_MASK;
gctUINT32 stlbEntryNum = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM;
stlbOffset = (Address & stlbMask) >> stlbShift;
pageTable = area->stlbLogical;
index = (mtlbOffset - area->mappingStart) * stlbEntryNum + stlbOffset;
*PageTable = pageTable + index;
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckMMU_GetAreaType(
IN gckMMU Mmu,
IN gctUINT32 GpuAddress,
OUT gceAREA_TYPE *AreaType
)
{
gctUINT32 mtlbIndex;
gctBOOL flatMapped;
gceSTATUS status = gcvSTATUS_OK;
gcsADDRESS_AREA_PTR area4K = &Mmu->dynamicArea4K;
#if gcdENABLE_GPU_1M_PAGE
gcsADDRESS_AREA_PTR area1M = &Mmu->dynamicArea1M;
#endif
mtlbIndex = _MtlbOffset(GpuAddress);
gcmkONERROR(gckMMU_IsFlatMapped(Mmu, gcvINVALID_PHYSICAL_ADDRESS, GpuAddress, &flatMapped));
if (flatMapped)
{
*AreaType = gcvAREA_TYPE_FLATMAP;
}
#if gcdENABLE_GPU_1M_PAGE
else if (mtlbIndex >= area1M->mappingStart && mtlbIndex <= area1M->mappingEnd)
{
*AreaType = gcvAREA_TYPE_1M;
}
#endif
else if (mtlbIndex >= area4K->mappingStart && mtlbIndex <= area4K->mappingEnd)
{
*AreaType = gcvAREA_TYPE_4K;
}
else
{
*AreaType = gcvAREA_TYPE_UNKNOWN;
}
OnError:
return status;
}