/****************************************************************************
*
*    The MIT License (MIT)
*
*    Copyright (c) 2014 - 2020 Vivante Corporation
*
*    Permission is hereby granted, free of charge, to any person obtaining a
*    copy of this software and associated documentation files (the "Software"),
*    to deal in the Software without restriction, including without limitation
*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
*    and/or sell copies of the Software, and to permit persons to whom the
*    Software is furnished to do so, subject to the following conditions:
*
*    The above copyright notice and this permission notice shall be included in
*    all copies or substantial portions of the Software.
*
*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
*    DEALINGS IN THE SOFTWARE.
*
*****************************************************************************
*
*    The GPL License (GPL)
*
*    Copyright (C) 2014 - 2020 Vivante Corporation
*
*    This program is free software; you can redistribute it and/or
*    modify it under the terms of the GNU General Public License
*    as published by the Free Software Foundation; either version 2
*    of the License, or (at your option) any later version.
*
*    This program is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU General Public License for more details.
*
*    You should have received a copy of the GNU General Public License
*    along with this program; if not, write to the Free Software Foundation,
*    Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*****************************************************************************
*
*    Note: This software is released under dual MIT and GPL licenses. A
*    recipient may use this file under the terms of either the MIT license or
*    GPL License. If you wish to use only one license not the other, you can
*    indicate your decision by deleting one of the above license notices in your
*    version of this file.
*
*****************************************************************************/


#include "gc_hal_kernel_precomp.h"

#define _GC_OBJ_ZONE    gcvZONE_MMU

typedef enum _gceMMU_TYPE
{
    gcvMMU_USED     = (0 << 4),
    gcvMMU_SINGLE   = (1 << 4),
    gcvMMU_FREE     = (2 << 4),
}
gceMMU_TYPE;

/* VIP SRAM start virtual address. */
#define gcdRESERVE_START (4 << 20)
#define gcdRESERVE_ALIGN (4 << 10)

#define gcmENTRY_TYPE(x) (x & 0xF0)

#define gcmENTRY_COUNT(x) ((x & 0xFFFFFF00) >> 8)

#define gcdMMU_TABLE_DUMP       0

#define gcdVERTEX_START      (128 << 10)

typedef struct _gcsMMU_STLB_CHUNK *gcsMMU_STLB_CHUNK_PTR;

typedef struct _gcsMMU_STLB_CHUNK
{
    gckVIDMEM_NODE  videoMem;
    gctUINT32_PTR   logical;
    gctSIZE_T       size;
    gctPHYS_ADDR_T  physBase;
    gctSIZE_T       pageCount;
    gctUINT32       mtlbIndex;
    gctUINT32       mtlbEntryNum;
    gcsMMU_STLB_CHUNK_PTR next;
} gcsMMU_STLB_CHUNK;

#if gcdSHARED_PAGETABLE
typedef struct _gcsSharedPageTable * gcsSharedPageTable_PTR;
typedef struct _gcsSharedPageTable
{
    /* Shared gckMMU object. */
    gckMMU          mmu;

    /* Hardwares which use this shared pagetable. */
    gckHARDWARE     hardwares[gcdMAX_GPU_COUNT];

    /* Number of cores use this shared pagetable. */
    gctUINT32       reference;
}
gcsSharedPageTable;

static gcsSharedPageTable_PTR sharedPageTable = gcvNULL;
#endif

typedef struct _gcsFreeSpaceNode * gcsFreeSpaceNode_PTR;
typedef struct _gcsFreeSpaceNode
{
    gctUINT32       start;
    gctINT32        entries;
}
gcsFreeSpaceNode;

#if gcdENDIAN_BIG

#  define _WritePageEntry(pageEntry, entryValue) \
    *(gctUINT32_PTR)(pageEntry) = gcmBSWAP32((gctUINT32)(entryValue))

#  define _ReadPageEntry(pageEntry) \
    gcmBSWAP32(*(gctUINT32_PTR)(pageEntry))

#else

#  define _WritePageEntry(pageEntry, entryValue) \
    *(gctUINT32_PTR)(pageEntry) = (gctUINT32)(entryValue)

#  define _ReadPageEntry(pageEntry) \
    *(gctUINT32_PTR)(pageEntry)

#endif

static gceSTATUS
_FillPageTable(
    IN gctUINT32_PTR PageTable,
    IN gctUINT32     PageCount,
    IN gctUINT32     EntryValue
)
{
    gctUINT i;

    for (i = 0; i < PageCount; i++)
    {
        _WritePageEntry(PageTable + i, EntryValue);
    }

    return gcvSTATUS_OK;
}

static gceSTATUS
_FillMap(
    IN gctUINT32_PTR Map,
    IN gctUINT32     PageCount,
    IN gctUINT32     EntryValue
)
{
    gctUINT i;

    for (i = 0; i < PageCount; i++)
    {
        Map[i] = EntryValue;
    }

    return gcvSTATUS_OK;
}

static gceSTATUS
_Link(
    IN gcsADDRESS_AREA_PTR Area,
    IN gctUINT32 Index,
    IN gctUINT32 Next
    )
{
    if (Index >= Area->stlbEntries)
    {
        /* Just move heap pointer. */
        Area->heapList = Next;
    }
    else
    {
        /* Address page table. */
        gctUINT32_PTR map = Area->mapLogical;

        /* Dispatch on node type. */
        switch (gcmENTRY_TYPE(map[Index]))
        {
        case gcvMMU_SINGLE:
            /* Set single index. */
            map[Index] = (Next << 8) | gcvMMU_SINGLE;
            break;

        case gcvMMU_FREE:
            /* Set index. */
            map[Index + 1] = Next;
            break;

        default:
            gcmkFATAL("MMU table correcupted at index %u!", Index);
            return gcvSTATUS_HEAP_CORRUPTED;
        }
    }

    /* Success. */
    return gcvSTATUS_OK;
}

static gceSTATUS
_AddFree(
    IN gcsADDRESS_AREA_PTR Area,
    IN gctUINT32 Index,
    IN gctUINT32 Node,
    IN gctUINT32 Count
    )
{
    gctUINT32_PTR map = Area->mapLogical;

    if (Count == 1)
    {
        /* Initialize a single page node. */
        map[Node] = (~((1U<<8)-1)) | gcvMMU_SINGLE;
    }
    else
    {
        /* Initialize the node. */
        map[Node + 0] = (Count << 8) | gcvMMU_FREE;
        map[Node + 1] = ~0U;
    }

    /* Append the node. */
    return _Link(Area, Index, Node);
}

static gceSTATUS
_Collect(
    IN gcsADDRESS_AREA_PTR Area
    )
{
    gctUINT32_PTR map = Area->mapLogical;
    gceSTATUS status;
    gctUINT32 i, previous, start = 0, count = 0;

    previous = Area->heapList = ~0U;
    Area->freeNodes = gcvFALSE;

    /* Walk the entire page table. */
    for (i = 0; i < Area->stlbEntries; ++i)
    {
        /* Dispatch based on type of page. */
        switch (gcmENTRY_TYPE(map[i]))
        {
        case gcvMMU_USED:
            /* Used page, so close any open node. */
            if (count > 0)
            {
                /* Add the node. */
                gcmkONERROR(_AddFree(Area, previous, start, count));

                /* Reset the node. */
                previous = start;
                count    = 0;
            }
            break;

        case gcvMMU_SINGLE:
            /* Single free node. */
            if (count++ == 0)
            {
                /* Start a new node. */
                start = i;
            }
            break;

        case gcvMMU_FREE:
            /* A free node. */
            if (count == 0)
            {
                /* Start a new node. */
                start = i;
            }

            /* Advance the count. */
            count += map[i] >> 8;

            /* Advance the index into the page table. */
            i     += (map[i] >> 8) - 1;
            break;

        default:
            gcmkFATAL("MMU page table correcupted at index %u!", i);
            return gcvSTATUS_HEAP_CORRUPTED;
        }
    }

    /* See if we have an open node left. */
    if (count > 0)
    {
        /* Add the node to the list. */
        gcmkONERROR(_AddFree(Area, previous, start, count));
    }

    gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_MMU,
                   "Performed a garbage collection of the MMU heap.");

    /* Success. */
    return gcvSTATUS_OK;

OnError:
    /* Return the staus. */
    return status;
}

static gctUINT32
_SetPage(gctUINT32 PageAddress, gctUINT32 PageAddressExt, gctBOOL Writable)
{
    gctUINT32 entry = PageAddress
                    /* AddressExt */
                    | (PageAddressExt << 4)
                    /* Ignore exception */
                    | (0 << 1)
                    /* Present */
                    | (1 << 0);

    if (Writable)
    {
        /* writable */
        entry |= (1 << 2);
    }
#if gcdUSE_MMU_EXCEPTION
    else
    {
        /* If this page is read only, set exception bit to make exception happens
        ** when writing to it. */
        entry |= gcdMMU_STLB_EXCEPTION;
    }
#endif

    return entry;
}

static gctUINT32
_MtlbOffset(
    gctUINT32 Address
    )
{
    return (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT;
}

gctUINT32
_AddressToIndex(
    IN gcsADDRESS_AREA_PTR Area,
    IN gctUINT32 Address
    )
{

    gctUINT32 stlbShift = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT;
    gctUINT32 stlbMask  = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_MASK : gcdMMU_STLB_4K_MASK;
    gctUINT32 stlbEntryNum = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM;
    gctUINT32 mtlbOffset = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT;
    gctUINT32 stlbOffset = (Address & stlbMask) >> stlbShift;

    return (mtlbOffset - Area->mappingStart) * stlbEntryNum + stlbOffset;
}

static gctUINT32_PTR
_StlbEntry(
    gcsADDRESS_AREA_PTR Area,
    gctUINT32 Address
    )
{
    gctUINT32 index = _AddressToIndex(Area, Address);

    return &Area->stlbLogical[index];
}

static gctBOOL
_IsRangeInsected(
    gctUINT64 baseAddress1,
    gctSIZE_T size1,
    gctUINT64 baseAddress2,
    gctSIZE_T size2
    )
{
    gctUINT64 endAddress1 = baseAddress1 + size1 - 1;
    gctUINT64 endAddress2 = baseAddress2 + size2 - 1;

    if (!size1 || !size2)
    {
        return gcvFALSE;
    }

    return (((baseAddress2 <= endAddress1) && (endAddress2 >= baseAddress1)) ||
            ((baseAddress1 <= endAddress2) && (endAddress1 >= baseAddress2)));
}

static gceSTATUS
_FillFlatMappingInMap(
    gcsADDRESS_AREA_PTR Area,
    gctUINT32 Index,
    gctUINT32 NumPages
    )
{
    gceSTATUS status;
    gctUINT32 i;
    gctBOOL gotIt = gcvFALSE;
    gctUINT32 index = Index;
    gctUINT32_PTR map = Area->mapLogical;
    gctUINT32 previous = ~0U;

    /* Find node which contains index. */
    for (i = 0; !gotIt && (i < Area->stlbEntries);)
    {
        gctUINT32 numPages;

        switch (gcmENTRY_TYPE(map[i]))
        {
        case gcvMMU_SINGLE:
            if (i == index)
            {
                gotIt = gcvTRUE;
            }
            else
            {
                previous = i;
                i = map[i] >> 8;
            }
            break;

        case gcvMMU_FREE:
            numPages = map[i] >> 8;
            if (index >= i && index + NumPages - 1 < i + numPages)
            {
                gotIt = gcvTRUE;
            }
            else
            {
                previous = i;
                i = map[i + 1];
            }
            break;

        case gcvMMU_USED:
            i++;
            break;

        default:
            gcmkFATAL("MMU table correcupted at index %u!", index);
            gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
        }
    }

    switch (gcmENTRY_TYPE(map[i]))
    {
    case gcvMMU_SINGLE:
        /* Unlink single node from free list. */
        gcmkONERROR(
            _Link(Area, previous, map[i] >> 8));
        break;

    case gcvMMU_FREE:
        /* Split the node. */
        {
            gctUINT32 start;
            gctUINT32 next = map[i+1];
            gctUINT32 total = map[i] >> 8;
            gctUINT32 countLeft = index - i;
            gctUINT32 countRight = total - countLeft - NumPages;

            if (countLeft)
            {
                start = i;
                _AddFree(Area, previous, start, countLeft);
                previous = start;
            }

            if (countRight)
            {
                start = index + NumPages;
                _AddFree(Area, previous, start, countRight);
                previous = start;
            }

            _Link(Area, previous, next);
        }
        break;
    }

    _FillMap(&map[index], NumPages, gcvMMU_USED);

    return gcvSTATUS_OK;
OnError:
    return status;
}

static gceSTATUS
_CollectFreeSpace(
    IN gckMMU Mmu,
    OUT gcsFreeSpaceNode_PTR *Array,
    OUT gctINT * Size
    )
{
    gceSTATUS status = gcvSTATUS_OK;
    gctPOINTER pointer = gcvNULL;
    gcsFreeSpaceNode_PTR array = gcvNULL;
    gcsFreeSpaceNode_PTR node = gcvNULL;
    gctINT size = 0;
    gctINT i = 0;

    for (i = 0; i < gcdMMU_MTLB_ENTRY_NUM; i++)
    {
        if (!Mmu->mtlbLogical[i])
        {
            if (!node)
            {
                /* This is the first entry of the free space. */
                node += 1;
                size++;

            }
        }
        else if (node)
        {
            /* Reset the start. */
            node = gcvNULL;
        }
    }

    /* Allocate memory for the array. */
    gcmkONERROR(gckOS_Allocate(Mmu->os,
                               gcmSIZEOF(*array) * size,
                               &pointer));

    array = (gcsFreeSpaceNode_PTR)pointer;
    node  = gcvNULL;

    for (i = 0, size = 0; i < gcdMMU_MTLB_ENTRY_NUM; i++)
    {
        if (!Mmu->mtlbLogical[i])
        {
            if (!node)
            {
                /* This is the first entry of the free space. */
                node = &array[size++];

                node->start   = i;
                node->entries = 0;
            }

            node->entries++;
        }
        else if (node)
        {
            /* Reset the start. */
            node = gcvNULL;
        }
    }

#if gcdMMU_TABLE_DUMP
    for (i = 0; i < size; i++)
    {
        gckOS_Print("%s(%d): [%d]: start=%d, entries=%d.\n",
                __FUNCTION__, __LINE__,
                i,
                array[i].start,
                array[i].entries);
    }
#endif

    *Array = array;
    *Size  = size;

    return gcvSTATUS_OK;

OnError:
    if (pointer != gcvNULL)
    {
        gckOS_Free(Mmu->os, pointer);
    }

    return status;
}

gceSTATUS
_GetMtlbFreeSpace(
    IN gckMMU Mmu,
    IN gctUINT32 NumEntries,
    OUT gctUINT32 *MtlbStart,
    OUT gctUINT32 *MtlbEnd
    )
{
    gceSTATUS status = gcvSTATUS_OK;
    gcsFreeSpaceNode_PTR nodeArray = gcvNULL;
    gctINT i, nodeArraySize = 0;
    gctINT numEntries = gcdMMU_MTLB_ENTRY_NUM;
    gctINT32 mStart = -1;
    gctINT32 mEnd = -1;

    gcmkONERROR(_CollectFreeSpace(Mmu, &nodeArray, &nodeArraySize));

    /* Find the smallest space for NumEntries */
    for (i = 0; i < nodeArraySize; i++)
    {
        if (nodeArray[i].entries < numEntries && NumEntries <= (gctUINT32)nodeArray[i].entries)
        {
            numEntries = nodeArray[i].entries;

            mStart = nodeArray[i].start;
            mEnd   = nodeArray[i].start + NumEntries - 1;
        }
    }

    if (mStart == -1 && mEnd == -1)
    {
        gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
    }

    *MtlbStart = (gctUINT32)mStart;
    *MtlbEnd   = (gctUINT32)mEnd;

OnError:
    if (nodeArray)
    {
        gckOS_Free(Mmu->os, (gctPOINTER)nodeArray);
    }

    return status;
}

static gcePOOL _GetPageTablePool(IN gckOS Os)
{
    gcePOOL pool = gcvPOOL_DEFAULT;
    gctUINT64 data = 0;
    gceSTATUS status;

    status = gckOS_QueryOption(Os, "mmuPageTablePool", &data);

    if (status  == gcvSTATUS_OK)
    {
        if (data == 1)
        {
            pool = gcvPOOL_VIRTUAL;
        }
    }

    return pool;
}

static gceSTATUS
_FillFlatMapping(
    IN gckMMU Mmu,
    IN gctUINT64 PhysBase,
    IN gctSIZE_T Size,
    IN gctBOOL   reserved,
    IN gctBOOL   ableToShift,
    OUT gctUINT32 *GpuBaseAddress
    )
{
    gceSTATUS status;
    gctUINT32 mtlb;
    gctUINT32 physBase;
    gckKERNEL kernel = Mmu->hardware->kernel;
    gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K;
    gctBOOL physicalRangeOverlapped = gcvFALSE;
    gctBOOL virtualRangeOverlapped = gcvFALSE;
    gctBOOL specificFlatMapping = gcvFALSE;
    gctBOOL needShiftMapping = gcvFALSE;
    gctUINT64 flatBase = PhysBase;
    gctUINT32 flatSize = (gctUINT32) Size;
    gctUINT64 base = flatBase;
    gctUINT64 end  = base + flatSize;
    gctUINT32 reqVirtualBase = 0;
    gctUINT32 flatVirtualBase = 0;
    gctUINT32 i;

    /************************ Get flat mapping type and range. ************************/
    {
        for (i = 0; i < Mmu->gpuPhysicalRangeCount; i++)
        {
            if (base < Mmu->gpuPhysicalRanges[i].start)
            {
                if (end > Mmu->gpuPhysicalRanges[i].start)
                {
                    physicalRangeOverlapped = gcvTRUE;
                    if (Mmu->gpuPhysicalRanges[i].flag == gcvFLATMAP_DIRECT)
                    {
                        /* Overlapped part is direct mapping, continue direct mapping */
                        end = Mmu->gpuPhysicalRanges[i].start;
                    }
                    else
                    {
                        /* Overlapped part is shift mapping, do entire shift mapping */
                        needShiftMapping = gcvTRUE;
                    }
                }

                flatSize = (gctUINT32) (end - base);
            }
            else if (end > Mmu->gpuPhysicalRanges[i].end)
            {
                if (base < Mmu->gpuPhysicalRanges[i].end)
                {
                    physicalRangeOverlapped = gcvTRUE;
                    if (Mmu->gpuPhysicalRanges[i].flag == gcvFLATMAP_DIRECT)
                    {
                        /* Overlapped part is direct mapping, continue direct mapping */
                        base = Mmu->gpuPhysicalRanges[i].end + 1;
                    }
                    else
                    {
                        /* Overlapped part is shift mapping, do entire shift mapping */
                        needShiftMapping = gcvTRUE;
                    }

                }

                flatBase = base;
                flatSize = (gctUINT32) (end - base);
            }
            else
            {
                /* it is already inside existing flat mapping ranges. */
                flatSize = 0;
            }

            if (flatSize == 0)
            {
                if (GpuBaseAddress)
                {
                    *GpuBaseAddress = (gctUINT32) PhysBase;
                }

                return gcvSTATUS_OK;
            }
        }
    }

    /* overwrite the orignal parameters */
    PhysBase = flatBase;
    physBase = (gctUINT32)flatBase;

    mtlb = _MtlbOffset(physBase);

    if (GpuBaseAddress)
    {
        reqVirtualBase = *GpuBaseAddress;
    }

    /*
     * if no partcial physical range overlap to request entire shift mapping,
     * it is specific shift mapping or directly mapping by default.
     */
    if (!needShiftMapping)
    {
        flatVirtualBase = reqVirtualBase ? reqVirtualBase : (gctUINT32)flatBase;
    }

    for (i = 0; i < Mmu->gpuAddressRangeCount; i++)
    {
        if (_IsRangeInsected(flatVirtualBase, flatSize,
            Mmu->gpuAddressRanges[i].start,  Mmu->gpuAddressRanges[i].size))
        {
            virtualRangeOverlapped = gcvTRUE;
        }
    }

    /* If gpu virtual range overlapped or gpu physical over 4G, still need entire shift mapping */
    if ((!physicalRangeOverlapped && virtualRangeOverlapped) ||
        PhysBase + flatSize - 1 > 0xffffffff)
    {
        needShiftMapping = gcvTRUE;
    }

    if (needShiftMapping && !ableToShift)
    {
        /*
         * Return without mapping any address.
         * By now, only physBase physSize could run here.
         */
        return gcvSTATUS_OK;
    }

    specificFlatMapping = (reqVirtualBase && !virtualRangeOverlapped && !physicalRangeOverlapped);

    /************************ Setup flat mapping in dynamic range. ****************/
    if (area->mappingStart != gcvINVALID_ADDRESS && mtlb >= area->mappingStart && mtlb < area->mappingEnd)
    {
        /* This path is useless now, keep it 4K page size */

        gctUINT32_PTR stlbEntry;

        stlbEntry = _StlbEntry(area, physBase);

        /* Must be aligned to page. */
        gcmkASSERT((flatSize & 0xFFF) == 0);

        for (i = 0; i < (flatSize / gcdMMU_PAGE_4K_SIZE); i++)
        {
            /* Flat mapping in page table. */
            _WritePageEntry(stlbEntry, _SetPage(physBase + i * gcdMMU_PAGE_4K_SIZE, 0, gcvTRUE));
#if gcdMMU_TABLE_DUMP
            gckOS_Print("%s(%d): insert MTLB[%d] STLB[%d]: %08x\n",
                __FUNCTION__, __LINE__,
                (physBase & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT,
                ((physBase & gcdMMU_STLB_4K_MASK) >> gcdMMU_STLB_4K_SHIFT) + i,
                _ReadPageEntry(stlbEntry));
#endif
            stlbEntry++;
        }

#if gcdDUMP_IN_KERNEL
        {
            gctPHYS_ADDR_T physical;
            gctUINT32 data = _SetPage(physBase, 0, gcvTRUE) & ~0xF;
            gctUINT32 step = (_SetPage(physBase + gcdMMU_PAGE_4K_SIZE, 0, gcvTRUE) & ~0xF) - data;
            gctUINT32 mask = _SetPage(physBase, 0, gcvTRUE) & 0xF;

            physical  = area->stlbPhysical + 4 * _AddressToIndex(area, physBase);

            gcmkDUMP(Mmu->os,
                     "#[mmu-stlb: flat-mapping in dynamic: 0x%08X - 0x%08X]",
                     physBase, physBase - 1 + flatSize);

            gcmkDUMP(Mmu->os,
                     "@[physical.step 0x%010llX 0x%08X 0x%08lX 0x%08X 0x%08X",
                     (unsigned long long)physical, data,
                     (unsigned long)(flatSize / gcdMMU_PAGE_4K_SIZE * sizeof(gctUINT32)),
                     step, mask);
        }
#endif

        /* Flat mapping in map. */
        _FillFlatMappingInMap(area, _AddressToIndex(area, physBase), flatSize / gcdMMU_PAGE_4K_SIZE);

        return gcvSTATUS_OK;
    }

    /************************ Setup flat mapping in non dynamic range. **************/
    {
        gctBOOL mutex = gcvFALSE;
        gctUINT32 physBaseExt = (gctUINT32) (PhysBase >> 32);
        gctUINT32 start = physBase & ~gcdMMU_PAGE_1M_MASK;
        gctUINT32 end = (gctUINT32) (physBase + flatSize - 1) & ~gcdMMU_PAGE_1M_MASK;
        gctUINT32 mStart = start >> gcdMMU_MTLB_SHIFT;
        gctUINT32 mEnd = end >> gcdMMU_MTLB_SHIFT;
        gctUINT32 sStart = (start & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;
        gctUINT32 sEnd = (end & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;
        gctPHYS_ADDR_T physical;
        gcsMMU_STLB_CHUNK_PTR newStlbChunk = gcvNULL;
        gctUINT32 stlbIndex = 0;
        gctUINT32 totalNewStlbs = 0;
        gctINT32 firstMtlbEntry = -1;
        gctUINT32 mtlbCurEntry;
        gcsMMU_STLB_CHUNK_PTR curStlbChunk = gcvNULL;
        gceFLATMAP_FLAG mapFlag = gcvFLATMAP_DIRECT;
        enum
        {
            COLOR_NONE   = 0,
            COLOR_RED    = 1, /* occupied entry */
            COLOR_BLUE   = 2, /* empty entry */
            COLOR_MAX    = COLOR_BLUE,
        } lastColor = COLOR_NONE;
        gctUINT32 colorNumber = 0;

        /* Grab the mutex. */
        gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
        mutex = gcvTRUE;

        if (needShiftMapping)
        {
            gctUINT32 mEntries;
            gctUINT32 sEntries;

            mEntries = (flatSize + (1 << gcdMMU_MTLB_SHIFT) - 1) / (1 << gcdMMU_MTLB_SHIFT);

            gcmkONERROR(_GetMtlbFreeSpace(Mmu, mEntries, &mStart, &mEnd));

            sStart = 0;
            sEntries = (flatSize + gcdMMU_PAGE_1M_SIZE - 1) / gcdMMU_PAGE_1M_SIZE;
            sEnd = (sEntries - 1) % gcdMMU_STLB_1M_ENTRY_NUM;
            mapFlag = gcvFLATMAP_SHIFT;
        }

        if (specificFlatMapping)
        {
            start    = reqVirtualBase & ~gcdMMU_PAGE_1M_MASK;
            end      = (reqVirtualBase + flatSize - 1) & ~gcdMMU_PAGE_1M_MASK;
            mStart   = start >> gcdMMU_MTLB_SHIFT;
            mEnd     = end >> gcdMMU_MTLB_SHIFT;
            sStart   = (start & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;
            sEnd     = (end & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;
            mapFlag  = gcvFLATMAP_SHIFT;
        }

        /* No matter direct mapping or shift mapping or specific mapping, store gpu virtual ranges */
        flatVirtualBase = (mStart << gcdMMU_MTLB_SHIFT)
                        | (sStart << gcdMMU_STLB_1M_SHIFT)
                        | (physBase & gcdMMU_PAGE_1M_MASK);

        /* Return GPU virtual base address if necessary */
        if (GpuBaseAddress)
        {
            *GpuBaseAddress = flatVirtualBase;
        }

        mtlbCurEntry = mStart;

        /* find all new stlbs, part of new flat mapping range may already have stlbs*/
        while (mtlbCurEntry <= mEnd)
        {
            if (*(Mmu->mtlbLogical + mtlbCurEntry) == 0)
            {
                if (lastColor != COLOR_BLUE)
                {
                    if (colorNumber < COLOR_MAX)
                    {
                        lastColor = COLOR_BLUE;
                        colorNumber++;
                    }
                    else
                    {
                        gcmkPRINT("There is a hole in new flat mapping range, which is not correct");
                    }
                }

                totalNewStlbs++;
                if (-1 == firstMtlbEntry)
                {
                    firstMtlbEntry = mtlbCurEntry;
                }
            }
            else
            {
                if (lastColor != COLOR_RED)
                {
                    if (colorNumber < COLOR_MAX)
                    {
                        lastColor = COLOR_RED;
                        colorNumber++;
                    }
                    else
                    {
                        gcmkPRINT("There is a hole in new flat mapping range, which is not correct");
                    }
                }
            }
            mtlbCurEntry++;
        }

        /* Need allocate a new chunk of stlbs */
        if (totalNewStlbs)
        {
            gcePOOL pool = Mmu->pool;
            gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS;

            gcmkONERROR(
                gckOS_Allocate(Mmu->os,
                               sizeof(struct _gcsMMU_STLB_CHUNK),
                               (gctPOINTER *)&newStlbChunk));

            newStlbChunk->mtlbEntryNum = totalNewStlbs;
            newStlbChunk->next = gcvNULL;
            newStlbChunk->videoMem = gcvNULL;
            newStlbChunk->logical = gcvNULL;
            newStlbChunk->size = gcdMMU_STLB_1M_SIZE * newStlbChunk->mtlbEntryNum;
            newStlbChunk->pageCount = 0;
            newStlbChunk->mtlbIndex = firstMtlbEntry;

#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
            allocFlag |= gcvALLOC_FLAG_CACHEABLE;
#endif

            allocFlag |= gcvALLOC_FLAG_4GB_ADDR;

            gcmkONERROR(gckKERNEL_AllocateVideoMemory(
                kernel,
                gcdMMU_STLB_1M_SIZE,
                gcvVIDMEM_TYPE_COMMAND,
                allocFlag | gcvALLOC_FLAG_4K_PAGES,
                &newStlbChunk->size,
                &pool,
                &newStlbChunk->videoMem));

            /* Lock for kernel side CPU access. */
            gcmkONERROR(gckVIDMEM_NODE_LockCPU(
                kernel,
                newStlbChunk->videoMem,
                gcvFALSE,
                gcvFALSE,
                (gctPOINTER *)&newStlbChunk->logical));

            gcmkONERROR(gckOS_ZeroMemory(newStlbChunk->logical, newStlbChunk->size));

            /* Get CPU physical address. */
            gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
                kernel,
                newStlbChunk->videoMem,
                0,
                &physical));

            gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(
                Mmu->os,
                physical,
                &physical));

            newStlbChunk->physBase = physical;
        }

        while (mStart <= mEnd)
        {
            gctUINT32 last = (mStart == mEnd) ? sEnd : (gcdMMU_STLB_1M_ENTRY_NUM - 1);
            gctPHYS_ADDR_T stlbPhyBase;
            gctUINT32_PTR stlbLogical;

            gcmkASSERT(mStart < gcdMMU_MTLB_ENTRY_NUM);

            if (*(Mmu->mtlbLogical + mStart) == 0)
            {
                gctUINT32 mtlbEntry;
                curStlbChunk = newStlbChunk;
                stlbPhyBase = curStlbChunk->physBase + (stlbIndex * gcdMMU_STLB_1M_SIZE);
                stlbLogical = (gctUINT32_PTR)((gctUINT8_PTR)curStlbChunk->logical + (stlbIndex * gcdMMU_STLB_1M_SIZE));

                physical  = stlbPhyBase
                          /* 1MB page size */
                          | (1 << 3)
                          /* Ignore exception */
                          | (0 << 1)
                          /* Present */
                          | (1 << 0);

                gcmkSAFECASTPHYSADDRT(mtlbEntry, physical);

                _WritePageEntry(Mmu->mtlbLogical + mStart, mtlbEntry);

#if gcdMMU_TABLE_DUMP
                gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n",
                    __FUNCTION__, __LINE__,
                    mStart,
                    _ReadPageEntry(Mmu->mtlbLogical + mStart));

                gckOS_Print("%s(%d): STLB: logical:%08x -> physical:%08x\n",
                        __FUNCTION__, __LINE__,
                        stlbLogical,
                        stlbPhyBase);
#endif

                gcmkDUMP(Mmu->os, "#[mmu-mtlb: flat-mapping, slot: %d]", mStart);

                gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08X]",
                         (unsigned long long)Mmu->mtlbPhysical + mStart * 4,
                         Mmu->mtlbLogical[mStart], 4);

                ++stlbIndex;
            }
            else
            {
                gctUINT32 mtlbEntry = _ReadPageEntry(Mmu->mtlbLogical + mStart);
                gctUINT stlbOffset;

                curStlbChunk = (gcsMMU_STLB_CHUNK_PTR)Mmu->staticSTLB;

                while (curStlbChunk)
                {
                    if ((mStart >= curStlbChunk->mtlbIndex) &&
                        (mStart < (curStlbChunk->mtlbIndex + curStlbChunk->mtlbEntryNum)))
                    {
                        break;
                    }
                    curStlbChunk = curStlbChunk->next;
                }
                gcmkASSERT(curStlbChunk);

                if (!curStlbChunk)
                    gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);

                stlbOffset = mStart - curStlbChunk->mtlbIndex;

                stlbPhyBase = curStlbChunk->physBase + (stlbOffset * gcdMMU_STLB_1M_SIZE);
                stlbLogical = (gctUINT32_PTR)((gctUINT8_PTR)curStlbChunk->logical + (stlbOffset * gcdMMU_STLB_1M_SIZE));
                if (stlbPhyBase != (mtlbEntry & gcdMMU_MTLB_ENTRY_STLB_MASK))
                {
                    gcmkASSERT(0);
                }
            }

#if gcdDUMP_IN_KERNEL
            /* Start index. */
            i = sStart;

            gcmkDUMP(Mmu->os, "#[mmu-stlb: flat-mapping: 0x%08X - 0x%08X]",
                     start, start + (last - sStart) * gcdMMU_PAGE_1M_SIZE - 1);
#endif

            while (sStart <= last)
            {
                gcmkASSERT(!(start & gcdMMU_PAGE_1M_MASK));
                if (reserved)
                {
                    /* program NOT_PRESENT | EXCEPTION  for reserved entries */
                    _WritePageEntry(stlbLogical + sStart, 1 << 1);
                }
                else
                {
                    _WritePageEntry(stlbLogical + sStart, _SetPage(start, physBaseExt, gcvTRUE));
                }
#if gcdMMU_TABLE_DUMP
                gckOS_Print("%s(%d): insert STLB[%d]: %08x\n",
                    __FUNCTION__, __LINE__,
                    sStart,
                    _ReadPageEntry(stlbLogical + sStart));
#endif
                /* next page. */
                start += gcdMMU_PAGE_1M_SIZE;
                if (start == 0)
                {
                    physBaseExt++;
                }
                sStart++;
                curStlbChunk->pageCount++;
            }

#if gcdDUMP_IN_KERNEL
            {
                gctUINT32 data = stlbLogical[i] & ~0xF;
                gctUINT32 step = (last > i) ? (stlbLogical[i + 1] - stlbLogical[i]) : 0;
                gctUINT32 mask = stlbLogical[i] & 0xF;

                gcmkDUMP(Mmu->os,
                         "@[physical.step 0x%010llX 0x%08X 0x%08X 0x%08X 0x%08X]",
                         (unsigned long long)stlbPhyBase + i * 4,
                         data, (last - i) * 4, step, mask);
            }
#endif

            gcmkONERROR(gckVIDMEM_NODE_CleanCache(
                kernel,
                curStlbChunk->videoMem,
                0,
                curStlbChunk->logical,
                curStlbChunk->size
                ));

            sStart = 0;
            ++mStart;
        }

        gcmkASSERT(totalNewStlbs == stlbIndex);

        if (newStlbChunk)
        {
            /* Insert the stlbChunk into staticSTLB. */
            if (Mmu->staticSTLB == gcvNULL)
            {
                Mmu->staticSTLB = newStlbChunk;
            }
            else
            {
                gcmkASSERT(newStlbChunk != gcvNULL);
                gcmkASSERT(newStlbChunk->next == gcvNULL);
                newStlbChunk->next = Mmu->staticSTLB;
                Mmu->staticSTLB = newStlbChunk;
            }
        }

        /* Release the mutex. */
        gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));

#if gcdENABLE_TRUST_APPLICATION
        if (Mmu->hardware->options.secureMode == gcvSECURE_IN_TA)
        {
            gckKERNEL_SecurityMapMemory(Mmu->hardware->kernel, gcvNULL, physBase, flatSize / gcdMMU_PAGE_4K_SIZE, &physBase);
        }
#endif

        /* Store the gpu physical ranges */
        Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].start = flatBase;
        Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].end   = flatBase + flatSize - 1;
        Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].size  = flatSize;
        Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].flag  = mapFlag;
        Mmu->gpuPhysicalRangeCount++;

        gcmkASSERT(Mmu->gpuPhysicalRangeCount <= gcdMAX_FLAT_MAPPING_COUNT);

        /* Store the gpu virtual ranges */
        Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].start = flatVirtualBase;
        Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].end   = flatVirtualBase + flatSize - 1;
        Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].size  = flatSize;
        Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].flag  = mapFlag;
        Mmu->gpuAddressRangeCount++;

        gcmkASSERT(Mmu->gpuAddressRangeCount <= gcdMAX_FLAT_MAPPING_COUNT);

        return gcvSTATUS_OK;
OnError:
        /* Roll back the allocation.
        ** We don't need roll back mtlb programming as gckmONERROR
        ** is only used during allocation time.
        */
        if (newStlbChunk)
        {
            if (newStlbChunk->videoMem)
            {
                gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(
                    kernel,
                    newStlbChunk->videoMem
                    ));
            }

            gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, newStlbChunk));
        }
        if (mutex)
        {
            /* Release the mutex. */
            gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
        }
        return status;
    }
}

static gceSTATUS
_SetupAddressArea(
    IN gckOS Os,
    IN gcsADDRESS_AREA_PTR Area,
    IN gctUINT32 NumMTLBEntries
    )
{
    gceSTATUS status;
    gctUINT32_PTR map;
    gctUINT32 stlbSize = (Area->areaType == gcvAREA_TYPE_1M)
                       ? gcdMMU_STLB_1M_SIZE : gcdMMU_STLB_4K_SIZE;

    gcmkHEADER();
    Area->stlbSize = NumMTLBEntries * stlbSize;

    gcmkSAFECASTSIZET(Area->stlbEntries, Area->stlbSize / gcmSIZEOF(gctUINT32));

    gcmkONERROR(gckOS_Allocate(Os, Area->stlbSize, (void **)&Area->mapLogical));

    /* Initialization. */
    map      = Area->mapLogical;
    map[0]   = (Area->stlbEntries << 8) | gcvMMU_FREE;
    map[1]   = ~0U;
    Area->heapList  = 0;
    Area->freeNodes = gcvFALSE;

    gcmkFOOTER_NO();
    return gcvSTATUS_OK;

OnError:
    gcmkFOOTER();
    return status;
}

static gceSTATUS
_ConstructDynamicStlb(
    IN gckMMU Mmu,
    IN gcsADDRESS_AREA_PTR Area,
    IN gctUINT32 NumEntries
    )
{
    gceSTATUS status = gcvSTATUS_OK;
    gctBOOL acquired = gcvFALSE;
    gckKERNEL kernel = Mmu->hardware->kernel;
    gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS;
    gcePOOL pool = Mmu->pool;
    gctUINT32 address;
    gctUINT32 mtlbEntry;
    gctUINT32 i;

#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
    allocFlag |= gcvALLOC_FLAG_CACHEABLE;
#endif

    allocFlag |= gcvALLOC_FLAG_4GB_ADDR;

    /* Construct Slave TLB. */
    gcmkONERROR(gckKERNEL_AllocateVideoMemory(
                kernel,
                64,
                gcvVIDMEM_TYPE_COMMAND,
                allocFlag | gcvALLOC_FLAG_4K_PAGES,
                &Area->stlbSize,
                &pool,
                &Area->stlbVideoMem));

    /* Lock for kernel side CPU access. */
    gcmkONERROR(gckVIDMEM_NODE_LockCPU(
                kernel,
                Area->stlbVideoMem,
                gcvFALSE,
                gcvFALSE,
                (gctPOINTER *)&Area->stlbLogical));

#if gcdUSE_MMU_EXCEPTION
    gcmkONERROR(_FillPageTable(Area->stlbLogical,
                               Area->stlbEntries,
                               /* Enable exception */
                               1 << 1));
#else
    /* Invalidate all entries. */
    gcmkONERROR(gckOS_ZeroMemory(Area->stlbLogical,
                Area->stlbSize));
#endif

    /* Get stlb table physical. */
    gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
                kernel,
                Area->stlbVideoMem,
                0,
                &Area->stlbPhysical));

    gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Mmu->os,
                  Area->stlbPhysical,
                  &Area->stlbPhysical));

    if (Area->areaType == gcvAREA_TYPE_1M)
    {
        gcmkDUMP(Mmu->os, "#[mmu: 1M page size dynamic space: 0x%08X - 0x%08X]",
                 (Area->mappingStart << gcdMMU_MTLB_SHIFT),
                 (Area->mappingEnd << gcdMMU_MTLB_SHIFT) - 1);
    }
    else
    {
        gcmkDUMP(Mmu->os, "#[mmu: 4K page size dynamic space: 0x%08X - 0x%08X]",
                 (Area->mappingStart << gcdMMU_MTLB_SHIFT),
                 (Area->mappingEnd << gcdMMU_MTLB_SHIFT) - 1);
    }

    gcmkDUMP(Mmu->os, "#[mmu-stlb]");

    gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08lX]",
             (unsigned long long)Area->stlbPhysical,
             Area->stlbLogical[0],
             (unsigned long)Area->stlbSize);

    gcmkSAFECASTPHYSADDRT(address, Area->stlbPhysical);

    /* Grab the mutex. */
    gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
    acquired = gcvTRUE;

    /* Map to Master TLB. */
    for (i = Area->mappingStart;
         i < Area->mappingStart + NumEntries;
         i++)
    {
        if (Area->areaType == gcvAREA_TYPE_1M)
        {
            mtlbEntry = address
                      /* 1M page size */
                      | (1 << 3)
                      /*Ignore exception */
                      | (0 << 1)
                      /* Present */
                      | (1 << 0);

            address += gcdMMU_STLB_1M_SIZE;
        }
        else
        {
            mtlbEntry = address
                      /* 4KB page size */
                      | (0 << 2)
                      /*Ignore exception */
                      | (0 << 1)
                      /* Present */
                      | (1 << 0);

            address += gcdMMU_STLB_4K_SIZE;
        }

        _WritePageEntry(Mmu->mtlbLogical + i, mtlbEntry);

#if gcdMMU_TABLE_DUMP
        gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n",
                __FUNCTION__, __LINE__,
                i,
                _ReadPageEntry(Mmu->mtlbLogical + i));
#endif
    }

    gcmkDUMP(Mmu->os, "#[mmu-mtlb: slot: %d - %d]",
             Area->mappingStart, Area->mappingEnd - 1);

#if gcdDUMP_IN_KERNEL
    {
        gctUINT32 data = Mmu->mtlbLogical[Area->mappingStart] & ~0x3F;
        gctUINT32 step = 0;
        gctUINT32 mask = Mmu->mtlbLogical[Area->mappingStart] & 0x3F;

        if (NumEntries > 1)
        {
            step = Mmu->mtlbLogical[Area->mappingStart + 1]
                 - Mmu->mtlbLogical[Area->mappingStart];
        }

        gcmkDUMP(Mmu->os,
                 "@[physical.step 0x%010llX 0x%08X 0x%08X 0x%08X 0x%08X]",
                 (unsigned long long)(Mmu->mtlbPhysical + Area->mappingStart * 4),
                 data, NumEntries * 4, step, mask);
    }
#endif

OnError:
    if (acquired)
    {
        /* Release the mutex. */
        gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
    }

    return status;
}

gceSTATUS
gckMMU_SetupDynamicSpace(
    IN gckMMU Mmu
    )
{
    gceSTATUS status;
    gcsFreeSpaceNode_PTR nodeArray = gcvNULL;
    gctINT i, nodeArraySize = 0;
    gctINT numEntries = 0;
    gckKERNEL kernel = Mmu->hardware->kernel;
    gcsADDRESS_AREA_PTR area4K = &Mmu->dynamicArea4K;
#if gcdENABLE_GPU_1M_PAGE
    gcsADDRESS_AREA_PTR area1M = &Mmu->dynamicArea1M;
    gctINT numEntries1M;
#endif

    /* Find all the free address space. */
    gcmkONERROR(_CollectFreeSpace(Mmu, &nodeArray, &nodeArraySize));

    for (i = 0; i < nodeArraySize; i++)
    {
        if (nodeArray[i].entries > numEntries)
        {
            area4K->mappingStart = nodeArray[i].start;
            numEntries           = nodeArray[i].entries;
            area4K->mappingEnd   = area4K->mappingStart + numEntries - 1;
        }
    }

    gckOS_Free(Mmu->os, (gctPOINTER)nodeArray);

#if gcdENABLE_TRUST_APPLICATION
    if (gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_SECURITY) == gcvSTATUS_TRUE)
    {
        /* Setup secure address area when needed. */
        gctUINT32 secureAreaSize = gcdMMU_SECURE_AREA_SIZE;
        gcsADDRESS_AREA_PTR secureArea = &Mmu->secureArea;

        gcmkASSERT(numEntries > (gctINT)secureAreaSize);

        secureArea->mappingStart = area4K->mappingStart
                                 + (numEntries - secureAreaSize);

        gcmkONERROR(_SetupAddressArea(Mmu->os, secureArea, secureAreaSize));

        numEntries -= secureAreaSize;
        area4K->mappingEnd -= secureAreaSize;
    }
#endif

#if gcdENABLE_GPU_1M_PAGE
    numEntries1M = numEntries >> 1;
    area1M->mappingStart = area4K->mappingStart + (numEntries - numEntries1M);
    area1M->mappingEnd   = area1M->mappingStart + numEntries1M - 1;
    area1M->areaType     = gcvAREA_TYPE_1M;
    numEntries           -= numEntries1M;
    area4K->mappingEnd   -= numEntries1M;

    gcmkONERROR(_SetupAddressArea(Mmu->os, area1M, numEntries1M));

    gcmkONERROR(_ConstructDynamicStlb(Mmu, area1M, numEntries1M));
#endif

    area4K->areaType = gcvAREA_TYPE_4K;

    /* Setup normal address area. */
    gcmkONERROR(_SetupAddressArea(Mmu->os, area4K, numEntries));

    gcmkONERROR(_ConstructDynamicStlb(Mmu, area4K, numEntries));

    return gcvSTATUS_OK;

OnError:
#if gcdENABLE_GPU_1M_PAGE
    if (area1M->mapLogical)
    {
        gcmkVERIFY_OK(
            gckOS_Free(Mmu->os, (gctPOINTER) area1M->mapLogical));
    }

    if (area1M->stlbVideoMem)
    {
        gcmkVERIFY_OK(
            gckVIDMEM_NODE_Dereference(kernel,
                                       area1M->stlbVideoMem));
    }
#endif

    if (area4K->mapLogical)
    {
        gcmkVERIFY_OK(
            gckOS_Free(Mmu->os, (gctPOINTER) area4K->mapLogical));
    }

    if (area4K->stlbVideoMem)
    {
        gcmkVERIFY_OK(
            gckVIDMEM_NODE_Dereference(kernel,
                                       area4K->stlbVideoMem));
    }

    return status;
}

gctUINT32
_GetPageCountOfUsedNode(
    gctUINT32_PTR Node
    )
{
    gctUINT32 count;

    count = gcmENTRY_COUNT(*Node);

    if ((count << 8) == (~((1U<<8)-1)))
    {
        count = 1;
    }

    return count;
}

static gcsADDRESS_AREA_PTR
_GetProcessArea(
    IN gckMMU Mmu,
    IN gcePAGE_TYPE PageType,
    IN gctBOOL Secure
    )
{
#if gcdENABLE_TRUST_APPLICATION
    if (Secure == gcvTRUE)
    {
        return &Mmu->secureArea;
    }
#endif

    if (PageType == gcvPAGE_TYPE_1M)
    {
        return &Mmu->dynamicArea1M;
    }
    else
    {
        return &Mmu->dynamicArea4K;
    }
}

/*******************************************************************************
**
**  _Construct
**
**  Construct a new gckMMU object.
**
**  INPUT:
**
**      gckKERNEL Kernel
**          Pointer to an gckKERNEL object.
**
**      gctSIZE_T MmuSize
**          Number of bytes for the page table.
**
**  OUTPUT:
**
**      gckMMU * Mmu
**          Pointer to a variable that receives the gckMMU object pointer.
*/
gceSTATUS
_Construct(
    IN gckKERNEL Kernel,
    IN gctSIZE_T MmuSize,
    OUT gckMMU * Mmu
    )
{
    gckOS os;
    gckHARDWARE hardware;
    gceSTATUS status;
    gckMMU mmu = gcvNULL;
    gctUINT32_PTR map;
    gctPOINTER pointer = gcvNULL;
    gctPHYS_ADDR_T physBase;
    gctSIZE_T physSize;
    gctPHYS_ADDR_T contiguousBase;
    gctSIZE_T contiguousSize = 0;
    gctPHYS_ADDR_T externalBase;
    gctSIZE_T externalSize = 0;
    gctUINT32 gpuAddress;
    gctPHYS_ADDR_T gpuPhysical;
    gcsADDRESS_AREA_PTR area = gcvNULL;
    gcePOOL pool;
    gctUINT64 data;
    gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS;
    gctUINT64 mmuEnabled;

    gcmkHEADER_ARG("Kernel=0x%x MmuSize=%lu", Kernel, MmuSize);

    /* Verify the arguments. */
    gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
    gcmkVERIFY_ARGUMENT(MmuSize > 0);
    gcmkVERIFY_ARGUMENT(Mmu != gcvNULL);

    /* Extract the gckOS object pointer. */
    os = Kernel->os;
    gcmkVERIFY_OBJECT(os, gcvOBJ_OS);

    /* Extract the gckHARDWARE object pointer. */
    hardware = Kernel->hardware;
    gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);

    /* Allocate memory for the gckMMU object. */
    gcmkONERROR(gckOS_Allocate(os, sizeof(struct _gckMMU), &pointer));

    gckOS_ZeroMemory(pointer, sizeof(struct _gckMMU));

    mmu = pointer;

    /* Initialize the gckMMU object. */
    mmu->object.type      = gcvOBJ_MMU;
    mmu->os               = os;
    mmu->hardware         = hardware;
    mmu->pageTableMutex   = gcvNULL;
    mmu->mtlbLogical      = gcvNULL;
    mmu->staticSTLB       = gcvNULL;
    mmu->enabled          = gcvFALSE;

    mmu->dynamicAreaSetuped = gcvFALSE;
    mmu->pool = _GetPageTablePool(mmu->os);
    gcsLIST_Init(&mmu->hardwareList);

    /* Use 4K page size for MMU version 0. */
    area = &mmu->dynamicArea4K;
    area->mapLogical  = gcvNULL;
    area->stlbLogical = gcvNULL;

    /* Create the page table mutex. */
    gcmkONERROR(gckOS_CreateMutex(os, &mmu->pageTableMutex));

    gcmkONERROR(gckOS_QueryOption(os, "mmu", &mmuEnabled));

    if (hardware->mmuVersion == 0)
    {
        area->stlbSize = MmuSize;

        /* Construct address space management table. */
        gcmkONERROR(gckOS_Allocate(mmu->os,
                                   area->stlbSize,
                                   &pointer));

        area->mapLogical = pointer;

        pool = mmu->pool;

#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
        allocFlag |= gcvALLOC_FLAG_CACHEABLE;
#endif

        /* Construct page table read by GPU. */
        gcmkONERROR(gckKERNEL_AllocateVideoMemory(
                    Kernel,
                    4096,
                    gcvVIDMEM_TYPE_COMMAND,
                    allocFlag,
                    &area->stlbSize,
                    &pool,
                    &area->stlbVideoMem));

        /* Lock for kernel side CPU access. */
        gcmkONERROR(gckVIDMEM_NODE_LockCPU(
                    Kernel,
                    area->stlbVideoMem,
                    gcvFALSE,
                    gcvFALSE,
                    &pointer));

        area->stlbLogical = pointer;

        /* Get CPU physical address. */
        gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
                    Kernel,
                    area->stlbVideoMem,
                    0,
                    &area->stlbPhysical));

        gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(mmu->os,
                    area->stlbPhysical,
                    &area->stlbPhysical));

        /* Compute number of entries in page table. */
        gcmkSAFECASTSIZET(area->stlbEntries, area->stlbSize / sizeof(gctUINT32));

        /* Mark all pages as free. */
        map      = area->mapLogical;

        _FillPageTable(area->stlbLogical, area->stlbEntries, mmu->safeAddress);

        gcmkDUMP(mmu->os,
                 "#[mmu0: fill with safe address]");

        gcmkDUMP(mmu->os,
                 "@[physical.fill 0x%010llX 0x%08X 0x%08lX]",
                 (unsigned long long)area->stlbPhysical,
                 area->stlbLogical[0],
                 (unsigned long)area->stlbSize);

        map[0] = (area->stlbEntries << 8) | gcvMMU_FREE;
        map[1] = ~0U;
        area->heapList  = 0;
        area->freeNodes = gcvFALSE;

        status = gckOS_QueryOption(mmu->os, "contiguousBase", &contiguousBase);

        if (gcmIS_SUCCESS(status))
        {
            status = gckOS_QueryOption(mmu->os, "contiguousSize", &data);
            contiguousSize = (gctSIZE_T)data;
        }

        if (gcmIS_SUCCESS(status) && contiguousSize)
        {
            /* Convert to GPU address. */
            mmu->contiguousBaseAddress = (gctUINT32)(contiguousBase - Kernel->hardware->baseAddress);
        }
    }
    else
    {
        mmu->mtlbSize = gcdMMU_MTLB_SIZE;

        pool = mmu->pool;

#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
        allocFlag |= gcvALLOC_FLAG_CACHEABLE;
#endif

        allocFlag |= gcvALLOC_FLAG_4GB_ADDR;

        /* 1K mode is 1024 byte aligned. */
        gcmkONERROR(gckKERNEL_AllocateVideoMemory(
                    Kernel,
                    1024,
                    gcvVIDMEM_TYPE_COMMAND,
                    allocFlag | gcvALLOC_FLAG_4K_PAGES,
                    &mmu->mtlbSize,
                    &pool,
                    &mmu->mtlbVideoMem));

        /* Lock for kernel side CPU access. */
        gcmkONERROR(gckVIDMEM_NODE_LockCPU(
                    Kernel,
                    mmu->mtlbVideoMem,
                    gcvFALSE,
                    gcvFALSE,
                    &pointer));

        mmu->mtlbLogical = pointer;

        mmu->dynamicArea4K.mappingStart = gcvINVALID_ADDRESS;
        mmu->dynamicArea1M.mappingStart = gcvINVALID_ADDRESS;

        /* Get mtlb table physical. */
        gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
                    Kernel,
                    mmu->mtlbVideoMem,
                    0,
                    &mmu->mtlbPhysical));

        gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(
                      mmu->os,
                      mmu->mtlbPhysical,
                      &mmu->mtlbPhysical));

        /* Invalid all the entries. */
        gcmkONERROR(
            gckOS_ZeroMemory(pointer, mmu->mtlbSize));

        gcmkONERROR(
            gckOS_QueryOption(mmu->os, "physBase", &physBase));

        gcmkONERROR(
            gckOS_QueryOption(mmu->os, "physSize", &data));
        physSize = (gctSIZE_T)data;

        gcmkONERROR(
            gckOS_CPUPhysicalToGPUPhysical(mmu->os, physBase, &gpuPhysical));

        gcmkSAFECASTPHYSADDRT(gpuAddress, gpuPhysical);

        if (physSize)
        {
            /* Setup user specified flat mapping. */
            gcmkONERROR(_FillFlatMapping(mmu, gpuAddress, physSize, gcvFALSE, gcvFALSE, gcvNULL));
        }

#if !(0 || gcdCAPTURE_ONLY_MODE)
        if (!_ReadPageEntry(mmu->mtlbLogical + 0))
        {
            gctUINT32 mtlbEntry;
            /*
             * Reserved the first mtlb.
             * 1MB page size, Ingore exception, Not Present.
             */
            mtlbEntry = (1 << 3)
                      | (0 << 1)
                      | (0 << 0);

            _WritePageEntry(mmu->mtlbLogical + 0, mtlbEntry);

            gcmkDUMP(mmu->os, "#[mmu-mtlb: reserved 16M space, slot: 0]");
            gcmkDUMP(mmu->os,
                     "@[physical.fill 0x%010llX 0x%08X 0x%08X]",
                     (unsigned long long)mmu->mtlbPhysical,
                     mmu->mtlbLogical[0], 4);

            /* Store the gpu virtual ranges */
            mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].start = 0;
            mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].end   = (16 << 20) - 1;
            mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].size  = (16 << 20);
            mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].flag  = gcvFLATMAP_DIRECT;
            mmu->gpuAddressRangeCount++;
        }
#endif

        status = gckOS_QueryOption(mmu->os, "contiguousBase", &contiguousBase);

        if (gcmIS_SUCCESS(status))
        {
            status = gckOS_QueryOption(mmu->os, "contiguousSize", &data);
            contiguousSize = (gctSIZE_T)data;
        }

        if (gcmIS_SUCCESS(status) && contiguousSize)
        {
            gctUINT64 gpuContiguousBase;
            gctUINT32 contiguousBaseAddress = 0;

            gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(mmu->os, contiguousBase, &gpuContiguousBase));

            /* Setup flat mapping for reserved memory (VIDMEM). */
            gcmkONERROR(_FillFlatMapping(mmu, gpuContiguousBase, contiguousSize, gcvFALSE, gcvTRUE, &contiguousBaseAddress));

            if (mmuEnabled)
            {
                mmu->contiguousBaseAddress = contiguousBaseAddress;
            }
            else
            {
                gcmkSAFECASTPHYSADDRT(mmu->contiguousBaseAddress, gpuContiguousBase);
            }
        }

        status = gckOS_QueryOption(mmu->os, "externalBase", &externalBase);

        if (gcmIS_SUCCESS(status))
        {
            status = gckOS_QueryOption(mmu->os, "externalSize", &data);
            externalSize = (gctSIZE_T)data;
        }

        if (gcmIS_SUCCESS(status) && externalSize)
        {
            gctUINT64 gpuExternalBase;
            gctUINT32 externalBaseAddress = 0;

            gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(mmu->os, externalBase, &gpuExternalBase));

            /* Setup flat mapping for external memory. */
            gcmkONERROR(_FillFlatMapping(mmu, gpuExternalBase, externalSize, gcvFALSE, gcvTRUE, &externalBaseAddress));

            mmu->externalBaseAddress = externalBaseAddress;
        }
    }

    /* A 64 byte for safe address, we use 256 here. */
    mmu->safePageSize = 256;

    pool = mmu->pool;

    /* Allocate safe page from video memory. */
    gcmkONERROR(gckKERNEL_AllocateVideoMemory(
        Kernel,
        256,
        gcvVIDMEM_TYPE_COMMAND,
        gcvALLOC_FLAG_CONTIGUOUS | gcvALLOC_FLAG_4K_PAGES | gcvALLOC_FLAG_4GB_ADDR,
        &mmu->safePageSize,
        &pool,
        &mmu->safePageVideoMem
        ));

    /* Lock for kernel side CPU access. */
    gcmkONERROR(gckVIDMEM_NODE_LockCPU(
        Kernel,
        mmu->safePageVideoMem,
        gcvFALSE,
        gcvFALSE,
        &mmu->safePageLogical
        ));

    /* Get CPU physical address. */
    gcmkONERROR(gckVIDMEM_NODE_GetPhysical(
        Kernel,
        mmu->safePageVideoMem,
        0,
        &mmu->safePagePhysical
        ));

    gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(
        os,
        mmu->safePagePhysical,
        &mmu->safePagePhysical
        ));

    gcmkSAFECASTPHYSADDRT(mmu->safeAddress, mmu->safePagePhysical);

    gckOS_ZeroMemory(mmu->safePageLogical, mmu->safePageSize);

    gcmkDUMP(mmu->os, "#[safe page]");
    gcmkDUMP(mmu->os,
            "@[physical.fill 0x%010llX 0x%08X 0x%08lX]",
            (unsigned long long)mmu->safePagePhysical,
            0,
            (unsigned long)mmu->safePageSize);

    gcmkDUMP_BUFFER(
        mmu->os,
        gcvDUMP_BUFFER_KERNEL_COMMAND,
        mmu->safePageLogical,
        mmu->safeAddress,
        mmu->safePageSize
        );

    gcmkONERROR(gckQUEUE_Allocate(os, &mmu->recentFreedAddresses, 16));

    mmu->sRAMMapped = gcvFALSE;

    /* Return the gckMMU object pointer. */
    *Mmu = mmu;

    /* Success. */
    gcmkFOOTER_ARG("*Mmu=0x%x", *Mmu);
    return gcvSTATUS_OK;

OnError:
    /* Roll back. */
    if (mmu != gcvNULL)
    {
        if (area != gcvNULL && area->mapLogical != gcvNULL)
        {
            gcmkVERIFY_OK(
                gckOS_Free(os, (gctPOINTER) area->mapLogical));

            gcmkVERIFY_OK(
                gckVIDMEM_NODE_Dereference(Kernel,
                                           area->stlbVideoMem));
        }

        if (mmu->mtlbLogical != gcvNULL)
        {
            gcmkVERIFY_OK(
                gckVIDMEM_NODE_Dereference(Kernel,
                                           mmu->mtlbVideoMem));
        }

        if (mmu->pageTableMutex != gcvNULL)
        {
            /* Delete the mutex. */
            gcmkVERIFY_OK(
                gckOS_DeleteMutex(os, mmu->pageTableMutex));
        }

        gcmkVERIFY_OK(gckQUEUE_Free(os, &mmu->recentFreedAddresses));

        /* Mark the gckMMU object as unknown. */
        mmu->object.type = gcvOBJ_UNKNOWN;

        /* Free the allocates memory. */
        gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, mmu));
    }

    /* Return the status. */
    gcmkFOOTER();
    return status;
}

static gceSTATUS
_FreeAddressArea(
    gckKERNEL Kernel,
    gcsADDRESS_AREA * Area
    )
{
    gceSTATUS status = gcvSTATUS_OK;

    if (Area->mapLogical != gcvNULL)
    {
        gcmkVERIFY_OK(
            gckOS_Free(Kernel->os, (gctPOINTER) Area->mapLogical));
    }

    if (Area->stlbLogical != gcvNULL)
    {
        /* Free page table. */
        gcmkVERIFY_OK(
            gckVIDMEM_NODE_Dereference(Kernel,
                                       Area->stlbVideoMem));
    }

    return status;
}

/*******************************************************************************
**
**  _Destroy
**
**  Destroy a gckMMU object.
**
**  INPUT:
**
**      gckMMU Mmu
**          Pointer to an gckMMU object.
**
**  OUTPUT:
**
**      Nothing.
*/
gceSTATUS
_Destroy(
    IN gckMMU Mmu
    )
{
    gckKERNEL kernel = Mmu->hardware->kernel;

    gcmkHEADER_ARG("Mmu=0x%x", Mmu);

    /* Verify the arguments. */
    gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);

    while (Mmu->staticSTLB != gcvNULL)
    {
        gcsMMU_STLB_CHUNK_PTR pre = Mmu->staticSTLB;
        Mmu->staticSTLB = pre->next;

        if (pre->videoMem)
        {
            gcmkVERIFY_OK(
                gckVIDMEM_NODE_Dereference(kernel,
                                           pre->videoMem));
        }

        if (pre->mtlbEntryNum != 0)
        {
            gctUINT i;
            for (i = 0; i < pre->mtlbEntryNum; ++i)
            {
                _WritePageEntry(Mmu->mtlbLogical + pre->mtlbIndex + i, 0);
#if gcdMMU_TABLE_DUMP
                gckOS_Print("%s(%d): clean MTLB[%d]\n",
                    __FUNCTION__, __LINE__,
                    pre->mtlbIndex + i);
#endif
            }

            gcmkDUMP(Mmu->os,
                     "#[mmu-mtlb: clean up slot: %d - %d]",
                     pre->mtlbIndex,
                     pre->mtlbIndex + pre->mtlbEntryNum - 1);

            gcmkDUMP(Mmu->os,
                     "@[physical.fill 0x%010llX 0x%08X 0x%08lX]",
                     (unsigned long long)(Mmu->mtlbPhysical + pre->mtlbIndex * 4),
                     Mmu->mtlbLogical[pre->mtlbIndex],
                     (unsigned long)(pre->mtlbEntryNum * 4));
        }

        gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, pre));
    }

    if (Mmu->hardware->mmuVersion != 0)
    {
        gcmkVERIFY_OK(
            gckVIDMEM_NODE_Dereference(kernel,
                                       Mmu->mtlbVideoMem));
    }

    /* Free address area. */
    gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->dynamicArea4K));
#if gcdENABLE_GPU_1M_PAGE
    gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->dynamicArea1M));
#endif
    gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->secureArea));

    /* Delete the page table mutex. */
    gcmkVERIFY_OK(gckOS_DeleteMutex(Mmu->os, Mmu->pageTableMutex));

    if (Mmu->safePageLogical != gcvNULL)
    {
        gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(
            kernel,
            Mmu->safePageVideoMem
            ));
    }

    gcmkVERIFY_OK(gckQUEUE_Free(Mmu->os, &Mmu->recentFreedAddresses));

    /* Mark the gckMMU object as unknown. */
    Mmu->object.type = gcvOBJ_UNKNOWN;

    /* Free the gckMMU object. */
    gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, Mmu));

    /* Success. */
    gcmkFOOTER_NO();
    return gcvSTATUS_OK;
}

/*******************************************************************************
** _AdjstIndex
**
**  Adjust the index from which we search for a usable node to make sure
**  index allocated is greater than Start.
*/
gceSTATUS
_AdjustIndex(
    IN gckMMU Mmu,
    IN gctUINT32 Index,
    IN gctUINT32 PageCount,
    IN gctUINT32 Start,
    OUT gctUINT32 * IndexAdjusted
    )
{
    gceSTATUS status;
    gctUINT32 index = Index;
    gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K;
    gctUINT32_PTR map = area->mapLogical;

    gcmkHEADER();

    for (; index < area->stlbEntries;)
    {
        gctUINT32 result = 0;
        gctUINT32 nodeSize = 0;

        if (index >= Start)
        {
            break;
        }

        switch (gcmENTRY_TYPE(map[index]))
        {
        case gcvMMU_SINGLE:
            nodeSize = 1;
            break;

        case gcvMMU_FREE:
            nodeSize = map[index] >> 8;
            break;

        default:
            gcmkFATAL("MMU table correcupted at index %u!", index);
            gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
        }

        if (nodeSize > PageCount)
        {
            result = index + (nodeSize - PageCount);

            if (result >= Start)
            {
                break;
            }
        }

        switch (gcmENTRY_TYPE(map[index]))
        {
        case gcvMMU_SINGLE:
            index = map[index] >> 8;
            break;

        case gcvMMU_FREE:
            index = map[index + 1];
            break;

        default:
            gcmkFATAL("MMU table correcupted at index %u!", index);
            gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
        }
    }

    *IndexAdjusted = index;

    gcmkFOOTER_NO();
    return gcvSTATUS_OK;

OnError:
    gcmkFOOTER();
    return status;
}

gceSTATUS
gckMMU_Construct(
    IN gckKERNEL Kernel,
    IN gctSIZE_T MmuSize,
    OUT gckMMU * Mmu
    )
{
#if gcdSHARED_PAGETABLE
    gceSTATUS status;
    gctPOINTER pointer;

    gcmkHEADER_ARG("Kernel=0x%08x", Kernel);

    if (sharedPageTable == gcvNULL)
    {
        gcmkONERROR(
                gckOS_Allocate(Kernel->os,
                               sizeof(struct _gcsSharedPageTable),
                               &pointer));
        sharedPageTable = pointer;

        gcmkONERROR(
                gckOS_ZeroMemory(sharedPageTable,
                    sizeof(struct _gcsSharedPageTable)));

        gcmkONERROR(_Construct(Kernel, MmuSize, &sharedPageTable->mmu));
    }

    *Mmu = sharedPageTable->mmu;

    sharedPageTable->hardwares[sharedPageTable->reference] = Kernel->hardware;

    sharedPageTable->reference++;

    gcmkFOOTER_ARG("sharedPageTable->reference=%lu", sharedPageTable->reference);
    return gcvSTATUS_OK;

OnError:
    if (sharedPageTable)
    {
        if (sharedPageTable->mmu)
        {
            gcmkVERIFY_OK(gckMMU_Destroy(sharedPageTable->mmu));
        }

        gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, sharedPageTable));
    }

    gcmkFOOTER();
    return status;
#else
    return _Construct(Kernel, MmuSize, Mmu);
#endif
}

gceSTATUS
gckMMU_Destroy(
    IN gckMMU Mmu
    )
{
#if gcdSHARED_PAGETABLE
    gckOS os = Mmu->os;

    sharedPageTable->reference--;

    if (sharedPageTable->reference == 0)
    {
        if (sharedPageTable->mmu)
        {
            gcmkVERIFY_OK(_Destroy(Mmu));
        }

        gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, sharedPageTable));
    }

    return gcvSTATUS_OK;
#else
    return _Destroy(Mmu);
#endif
}

/*******************************************************************************
**
**  gckMMU_AllocatePages
**
**  Allocate pages inside the page table.
**
**  INPUT:
**
**      gckMMU Mmu
**          Pointer to an gckMMU object.
**
**      gctSIZE_T PageCount
**          Number of pages to allocate.
**
**  OUTPUT:
**
**      gctPOINTER * PageTable
**          Pointer to a variable that receives the base address of the page
**          table.
**
**      gctUINT32 * Address
**          Pointer to a variable that receives the hardware specific address.
*/
gceSTATUS
_AllocatePages(
    IN gckMMU Mmu,
    IN gctSIZE_T PageCount,
    IN gceVIDMEM_TYPE Type,
    IN gcePAGE_TYPE PageType,
    IN gctBOOL Secure,
    OUT gctPOINTER * PageTable,
    OUT gctUINT32 * Address
    )
{
    gceSTATUS status;
    gctBOOL mutex = gcvFALSE;
    gctUINT32 index = 0, previous = ~0U, left;
    gctUINT32_PTR map;
    gctBOOL gotIt;
    gctUINT32 address;
    gctUINT32 pageCount;
    gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, Secure);

    gcmkHEADER_ARG("Mmu=0x%x PageCount=%lu", Mmu, PageCount);

    /* Verify the arguments. */
    gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
    gcmkVERIFY_ARGUMENT(PageCount > 0);
    gcmkVERIFY_ARGUMENT(PageTable != gcvNULL);

    if (PageCount > area->stlbEntries)
    {
        /* Not enough pages avaiable. */
        gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
    }

    gcmkSAFECASTSIZET(pageCount, PageCount);

#if gcdBOUNDARY_CHECK
    /* Extra pages as bounary. */
    pageCount += gcdBOUNDARY_CHECK * 2;
#endif

    /* Grab the mutex. */
    gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
    mutex = gcvTRUE;

    /* Cast pointer to page table. */
    for (map = area->mapLogical, gotIt = gcvFALSE; !gotIt;)
    {
        index = area->heapList;

        if ((Mmu->hardware->mmuVersion == 0) &&
            (Type == gcvVIDMEM_TYPE_VERTEX_BUFFER))
        {
            gcmkONERROR(_AdjustIndex(
                Mmu,
                index,
                pageCount,
                gcdVERTEX_START / gcmSIZEOF(gctUINT32),
                &index
                ));
        }

        /* Walk the heap list. */
        for (; !gotIt && (index < area->stlbEntries);)
        {
            /* Check the node type. */
            switch (gcmENTRY_TYPE(map[index]))
            {
            case gcvMMU_SINGLE:
                /* Single odes are valid if we only need 1 page. */
                if (pageCount == 1)
                {
                    gotIt = gcvTRUE;
                }
                else
                {
                    /* Move to next node. */
                    previous = index;
                    index    = map[index] >> 8;
                }
                break;

            case gcvMMU_FREE:
                /* Test if the node has enough space. */
                if (pageCount <= (map[index] >> 8))
                {
                    gotIt = gcvTRUE;
                }
                else
                {
                    /* Move to next node. */
                    previous = index;
                    index    = map[index + 1];
                }
                break;

            default:
                gcmkFATAL("MMU table correcupted at index %u!", index);
                gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
            }
        }

        /* Test if we are out of memory. */
        if (index >= area->stlbEntries)
        {
            if (area->freeNodes)
            {
                /* Time to move out the trash! */
                gcmkONERROR(_Collect(area));

                /* We are going to search from start, so reset previous to start. */
                previous = ~0U;
            }
            else
            {
                /* Out of resources. */
                gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
            }
        }
    }

    switch (gcmENTRY_TYPE(map[index]))
    {
    case gcvMMU_SINGLE:
        /* Unlink single node from free list. */
        gcmkONERROR(
            _Link(area, previous, map[index] >> 8));
        break;

    case gcvMMU_FREE:
        /* Check how many pages will be left. */
        left = (map[index] >> 8) - pageCount;
        switch (left)
        {
        case 0:
            /* The entire node is consumed, just unlink it. */
            gcmkONERROR(
                _Link(area, previous, map[index + 1]));
            break;

        case 1:
            /* One page will remain.  Convert the node to a single node and
            ** advance the index. */
            map[index] = (map[index + 1] << 8) | gcvMMU_SINGLE;
            index ++;
            break;

        default:
            /* Enough pages remain for a new node.  However, we will just adjust
            ** the size of the current node and advance the index. */
            map[index] = (left << 8) | gcvMMU_FREE;
            index += left;
            break;
        }
        break;
    }

    /* Mark node as used. */
    gcmkONERROR(_FillMap(&map[index], pageCount, gcvMMU_USED));

#if gcdBOUNDARY_CHECK
    index += gcdBOUNDARY_CHECK;
#endif

    /* Record pageCount of allocated node at the beginning of node. */
    if (pageCount == 1)
    {
        map[index] = (~((1U<<8)-1)) | gcvMMU_USED;
    }
    else
    {
        map[index] = (pageCount << 8) | gcvMMU_USED;
    }

    if (area->stlbLogical != gcvNULL)
    {
        /* Return pointer to page table. */
        *PageTable = &area->stlbLogical[index];
    }
    else
    {
        /* Page table for secure area is handled in trust application. */
        *PageTable = gcvNULL;
    }

    /* Build virtual address. */
    if (Mmu->hardware->mmuVersion == 0)
    {
        gcmkONERROR(
                gckHARDWARE_BuildVirtualAddress(Mmu->hardware, index, 0, &address));
    }
    else
    {
        gctUINT32 num = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM;
        gctUINT32 shift = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT;
        gctUINT32 masterOffset = index / num + area->mappingStart;
        gctUINT32 slaveOffset = index % num;

        address = (masterOffset << gcdMMU_MTLB_SHIFT)
                | (slaveOffset << shift);
    }

    if (Address != gcvNULL)
    {
        *Address = address;
    }

    /* Release the mutex. */
    gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));

    /* Success. */
    gcmkFOOTER_ARG("*PageTable=0x%x *Address=%08x",
                   *PageTable, gcmOPT_VALUE(Address));
    return gcvSTATUS_OK;

OnError:

    if (mutex)
    {
        /* Release the mutex. */
        gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
    }

    /* Return the status. */
    gcmkFOOTER();
    return status;
}

/*******************************************************************************
**
**  gckMMU_FreePages
**
**  Free pages inside the page table.
**
**  INPUT:
**
**      gckMMU Mmu
**          Pointer to an gckMMU object.
**
**      gctPOINTER PageTable
**          Base address of the page table to free.
**
**      gctSIZE_T PageCount
**          Number of pages to free.
**
**  OUTPUT:
**
**      Nothing.
*/
gceSTATUS
_FreePages(
    IN gckMMU Mmu,
    IN gctBOOL Secure,
    IN gcePAGE_TYPE PageType,
    IN gctUINT32 Address,
    IN gctPOINTER PageTable,
    IN gctSIZE_T PageCount
    )
{
    gctUINT32 index;
    gctUINT32_PTR node;
    gceSTATUS status;
    gctBOOL acquired = gcvFALSE;
    gctUINT32 pageCount;
    gcuQUEUEDATA data;
    gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, gcvFALSE);
    gctUINT32 pageSize = (PageType == gcvPAGE_TYPE_1M)
                       ? gcdMMU_PAGE_1M_SIZE : gcdMMU_PAGE_4K_SIZE;

    gcmkHEADER_ARG("Mmu=0x%x PageTable=0x%x PageCount=%lu",
                   Mmu, PageTable, PageCount);

    /* Verify the arguments. */
    gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
    gcmkVERIFY_ARGUMENT(PageCount > 0);

    gcmkSAFECASTSIZET(pageCount, PageCount);

#if gcdBOUNDARY_CHECK
    pageCount += gcdBOUNDARY_CHECK * 2;
#endif

    /* Get the node by index. */
    index = (gctUINT32)((gctUINT32_PTR)PageTable - area->stlbLogical);

    node = area->mapLogical + index;

    if (pageCount != _GetPageCountOfUsedNode(node))
    {
        gcmkONERROR(gcvSTATUS_INVALID_REQUEST);
    }

#if gcdBOUNDARY_CHECK
    node -= gcdBOUNDARY_CHECK;
#endif

    gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
    acquired = gcvTRUE;

    if (Mmu->hardware->mmuVersion == 0)
    {
        _FillPageTable(PageTable, pageCount, Mmu->safeAddress);
    }

    if (pageCount == 1)
    {
       /* Single page node. */
        node[0] = (~((1U<<8)-1)) | gcvMMU_SINGLE;

        if (PageTable != gcvNULL)
        {
#if gcdUSE_MMU_EXCEPTION
        /* Enable exception */
        _WritePageEntry(PageTable, (1 << 1));
#else
        _WritePageEntry(PageTable, 0);
#endif
        }
    }
    else
    {
        /* Mark the node as free. */
        node[0] = (pageCount << 8) | gcvMMU_FREE;
        node[1] = ~0U;

        if (PageTable != gcvNULL)
        {
#if gcdUSE_MMU_EXCEPTION
            /* Enable exception */
            gcmkVERIFY_OK(_FillPageTable(PageTable, (gctUINT32)PageCount, 1 << 1));
#else
            gcmkVERIFY_OK(_FillPageTable(PageTable, (gctUINT32)PageCount, 0));
#endif
        }
    }

    gcmkDUMP(Mmu->os,
             "#[mmu-stlb: free 0x%08X - 0x%08X]",
             Address, Address + pageCount * pageSize - 1);

    gcmkDUMP(Mmu->os,
             "@[physical.fill 0x%010llX 0x%08X 0x%08X]",
             (unsigned long long)(area->stlbPhysical + index * 4),
             *(gctUINT32_PTR)PageTable,
             pageCount * 4);

    /* We have free nodes. */
    area->freeNodes = gcvTRUE;

    /* Record freed address range. */
    data.addressData.start = Address;
    data.addressData.end = Address + (gctUINT32)PageCount * pageSize;
    gckQUEUE_Enqueue(&Mmu->recentFreedAddresses, &data);

    gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
    acquired = gcvFALSE;

#if gcdENABLE_TRUST_APPLICATION
    if (Mmu->hardware->options.secureMode == gcvSECURE_IN_TA)
    {
        gckKERNEL_SecurityUnmapMemory(Mmu->hardware->kernel, Address, (gctUINT32)PageCount);
    }
#endif

    /* Success. */
    gcmkFOOTER_NO();
    return gcvSTATUS_OK;

OnError:
    if (acquired)
    {
        gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
    }

    gcmkFOOTER();
    return status;
}

gceSTATUS
gckMMU_AllocatePages(
    IN gckMMU Mmu,
    IN gctSIZE_T PageCount,
    IN gcePAGE_TYPE PageType,
    OUT gctPOINTER * PageTable,
    OUT gctUINT32 * Address
    )
{
    return gckMMU_AllocatePagesEx(
                Mmu, PageCount, gcvVIDMEM_TYPE_GENERIC, PageType, gcvFALSE, PageTable, Address);
}

gceSTATUS
gckMMU_AllocatePagesEx(
    IN gckMMU Mmu,
    IN gctSIZE_T PageCount,
    IN gceVIDMEM_TYPE Type,
    IN gcePAGE_TYPE PageType,
    IN gctBOOL Secure,
    OUT gctPOINTER * PageTable,
    OUT gctUINT32 * Address
    )
{
#if gcdDISABLE_GPU_VIRTUAL_ADDRESS
    gcmkPRINT("GPU virtual address is disabled.");
    return gcvSTATUS_NOT_SUPPORTED;
#else
    return _AllocatePages(Mmu, PageCount, Type, PageType, Secure, PageTable, Address);
#endif
}

gceSTATUS
gckMMU_FreePages(
    IN gckMMU Mmu,
    IN gctBOOL Secure,
    IN gcePAGE_TYPE PageType,
    IN gctUINT32 Address,
    IN gctPOINTER PageTable,
    IN gctSIZE_T PageCount
    )
{
    return _FreePages(Mmu, Secure, PageType, Address, PageTable, PageCount);
}

gceSTATUS
gckMMU_SetPage(
    IN gckMMU Mmu,
    IN gctPHYS_ADDR_T PageAddress,
    IN gcePAGE_TYPE PageType,
    IN gctBOOL Writable,
    IN gctUINT32 *PageEntry
    )
{
    gctUINT32 addressExt;
    gctUINT32 address;

    gcmkHEADER_ARG("Mmu=0x%x", Mmu);

    /* Verify the arguments. */
    gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
    gcmkVERIFY_ARGUMENT(PageEntry != gcvNULL);

    if (PageType == gcvPAGE_TYPE_1M)
    {
        gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFFFF));
    }
    else
    {
        gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFF));
    }

    /* [31:0]. */
    address    = (gctUINT32)(PageAddress & 0xFFFFFFFF);
    /* [39:32]. */
    addressExt = (gctUINT32)((PageAddress >> 32) & 0xFF);

    if (Mmu->hardware->mmuVersion == 0)
    {
        _WritePageEntry(PageEntry, address);
    }
    else
    {
        _WritePageEntry(PageEntry, _SetPage(address, addressExt, gcvTRUE));
    }

#ifdef DUMP_IN_KERNEL
    {
        gctUINT32 *stlbLogical = (PageType == gcvPAGE_TYPE_1M)
                               ? Mmu->dynamicArea1M.stlbLogical
                               : Mmu->dynamicArea4K.stlbLogical;

        gctPHYS_ADDR_T stlbPhysical = (PageType == gcvPAGE_TYPE_1M)
                                    ? Mmu->dynamicArea1M.stlbPhysical
                                    : Mmu->dynamicArea4K.stlbPhysical;

        gctPHYS_ADDR_T physical = stlbPhysical + (stlbLogical - PageEntry) * 4;

        gckDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 4",
                physical, *PageEntry);
    }
#endif

    /* Success. */
    gcmkFOOTER_NO();
    return gcvSTATUS_OK;
}

gceSTATUS
gckMMU_Flush(
    IN gckMMU Mmu,
    IN gceVIDMEM_TYPE Type
    )
{
    gckHARDWARE hardware;
    gctUINT32 mask;
    gctINT i;

    if (Type == gcvVIDMEM_TYPE_VERTEX_BUFFER ||
        Type == gcvVIDMEM_TYPE_INDEX_BUFFER ||
        Type == gcvVIDMEM_TYPE_COMMAND)
    {
        mask = gcvPAGE_TABLE_DIRTY_BIT_FE;
    }
    else
    {
        mask = gcvPAGE_TABLE_DIRTY_BIT_OTHER;
    }

#if gcdSHARED_PAGETABLE
    for (i = 0; i < gcdMAX_GPU_COUNT; i++)
    {
        gctUINT j;
        hardware = sharedPageTable->hardwares[i];
        if (hardware)
        {
            for (j = 0; j < gcvENGINE_GPU_ENGINE_COUNT; j++)
            {
                gcmkVERIFY_OK(gckOS_AtomSetMask(hardware->pageTableDirty[j], mask));
            }
        }
    }
#else
    hardware = Mmu->hardware;

    for (i = 0 ; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
    {
        gcmkVERIFY_OK(
            gckOS_AtomSetMask(hardware->pageTableDirty[i], mask));
    }

    {
        gcsLISTHEAD_PTR hardwareHead;
        gcmkLIST_FOR_EACH(hardwareHead, &Mmu->hardwareList)
        {
            hardware = gcmCONTAINEROF(hardwareHead, struct _gckHARDWARE, mmuHead);

            if (hardware != Mmu->hardware)
            {
                for (i = 0 ; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
                {
                    gcmkVERIFY_OK(
                        gckOS_AtomSetMask(hardware->pageTableDirty[i], mask));
                }
            }
        }
    }
#endif

    return gcvSTATUS_OK;
}

gceSTATUS
gckMMU_DumpPageTableEntry(
    IN gckMMU Mmu,
    IN gceAREA_TYPE AreaType,
    IN gctUINT32 Address
    )
{
    gctUINT32_PTR pageTable;
    gctUINT32 index;
    gctUINT32 mtlb, stlb;
    gcsADDRESS_AREA_PTR area = (AreaType == gcvAREA_TYPE_4K)
                             ? &Mmu->dynamicArea4K : &Mmu->dynamicArea1M;

    gctUINT32 stlbShift = (AreaType == gcvAREA_TYPE_4K)
                        ? gcdMMU_STLB_4K_SHIFT : gcdMMU_STLB_1M_SHIFT;

    gctUINT32 stlbMask  = (AreaType == gcvAREA_TYPE_4K)
                        ? gcdMMU_STLB_4K_MASK : gcdMMU_STLB_1M_MASK;

    gctUINT32 stlbEntryNum = (AreaType == gcvAREA_TYPE_4K)
                           ? gcdMMU_STLB_4K_ENTRY_NUM : gcdMMU_STLB_1M_ENTRY_NUM;

    gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address);
    gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);

    gcmkASSERT(Mmu->hardware->mmuVersion > 0);

    mtlb = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT;

    if (AreaType != gcvAREA_TYPE_FLATMAP)
    {
        stlb = (Address & stlbMask) >> stlbShift;

        pageTable = area->stlbLogical;

        index = (mtlb - area->mappingStart)
              * stlbEntryNum
              + stlb;

        gcmkPRINT("    Page table entry = 0x%08X", _ReadPageEntry(pageTable + index));
    }
    else
    {
        gcsMMU_STLB_CHUNK_PTR stlbChunkObj = Mmu->staticSTLB;
        gctUINT32 entry = Mmu->mtlbLogical[mtlb];

        stlb = (Address & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT;

        entry &= 0xFFFFFFF0;

        while (stlbChunkObj)
        {
            gctUINT i;
            gctBOOL found = gcvFALSE;
            for (i = 0; i < stlbChunkObj->mtlbEntryNum; ++i)
            {
                gctPHYS_ADDR_T stlbPhysBase = stlbChunkObj->physBase + (i * gcdMMU_STLB_1M_SIZE);
                gctUINT32_PTR stlbLogical =
                    (gctUINT32_PTR)((gctUINT8_PTR)stlbChunkObj->logical + (i * gcdMMU_STLB_1M_SIZE));
                if (entry == stlbPhysBase)
                {
                    gcmkPRINT("    Page table entry = 0x%08X", stlbLogical[stlb]);
                    found = gcvTRUE;
                    break;
                }
            }
            if (found)
                break;
            stlbChunkObj = stlbChunkObj->next;
        }
    }

    gcmkFOOTER_NO();
    return gcvSTATUS_OK;
}

void
gckMMU_CheckSaftPage(
    IN gckMMU Mmu
    )
{
    gctUINT8_PTR safeLogical = Mmu->safePageLogical;
    gctUINT32 offsets[] = {
        0,
        64,
        128,
        256,
        2560,
        4000
    };

    gctUINT32 i = 0;

    while (i < gcmCOUNTOF(offsets))
    {
        if (safeLogical[offsets[i]] != 0)
        {
            gcmkPRINT("%s(%d) safe page is over written [%d] = %x",
                      __FUNCTION__, __LINE__, i, safeLogical[offsets[i]]);
        }
    }
}

void
gckMMU_DumpAddressSpace(
    IN gckMMU Mmu
    )
{
    gctUINT i;
    gctUINT next;
    /* TODO: */
    gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K;
    gctUINT32_PTR map = area->mapLogical;
    gctBOOL used = gcvFALSE;
    gctUINT32 numPages;

    /* Grab the mutex. */
    gcmkVERIFY_OK(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));

    /* Find node which contains index. */
    for (i = 0; i < area->stlbEntries; i = next)
    {
        switch (gcmENTRY_TYPE(map[i]))
        {
        case gcvMMU_SINGLE:
            numPages = 1;
            next = i + numPages;
            used = gcvFALSE;
            break;

        case gcvMMU_FREE:
            numPages = map[i] >> 8;
            next = i + numPages;
            used = gcvFALSE;
            break;

        case gcvMMU_USED:
            numPages = 1;
            next = i + numPages;
            used = gcvTRUE;
            break;

        default:
            gcmkFATAL("MMU table correcupted at index %u!", i);
            return;
        }

        if (!used)
        {
            gcmkPRINT("Available Range [%d - %d)", i, i + numPages);
        }
    }

    /* Release the mutex. */
    gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));

}

void
gckMMU_DumpRecentFreedAddress(
    IN gckMMU Mmu
    )
{
    gckQUEUE queue = &Mmu->recentFreedAddresses;
    gctUINT32 i;
    gcuQUEUEDATA *data;

    if (queue->count)
    {
        gcmkPRINT("    Recent %d freed GPU address ranges:", queue->count);

        for (i = 0; i < queue->count; i++)
        {
            gckQUEUE_GetData(queue, i, &data);

            gcmkPRINT("      [%08X - %08X]", data->addressData.start, data->addressData.end);
        }
    }
}

gceSTATUS
gckMMU_FillFlatMapping(
    IN gckMMU Mmu,
    IN gctUINT64 PhysBase,
    IN gctSIZE_T Size
    )
{
    gceSTATUS status;
    gckHARDWARE hardware = Mmu->hardware;

    if (hardware->mmuVersion)
    {
        gcmkONERROR(_FillFlatMapping(Mmu, PhysBase, Size, gcvFALSE, gcvTRUE, gcvNULL));
    }

    return gcvSTATUS_OK;

OnError:
    return status;
}

gceSTATUS
gckMMU_IsFlatMapped(
    IN gckMMU Mmu,
    IN gctUINT64 Physical,
    IN gctUINT32 Address,
    OUT gctBOOL *In
    )
{
    gceSTATUS status;
    gctUINT32 i;
    gctBOOL inFlatmapping = gcvFALSE;

    gcmkHEADER();

    gcmkVERIFY_ARGUMENT(In != gcvNULL);

    if (gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_MMU) == gcvFALSE)
    {
        gcmkONERROR(gcvSTATUS_NOT_SUPPORTED);
    }

    if (Physical != gcvINVALID_PHYSICAL_ADDRESS)
    {
        for (i = 0; i < Mmu->gpuPhysicalRangeCount; i++)
        {
            if ((Physical >= Mmu->gpuPhysicalRanges[i].start) &&
                (Physical <= Mmu->gpuPhysicalRanges[i].end))
            {
                inFlatmapping = gcvTRUE;
                break;
            }
        }
    }

    if (Address != gcvINVALID_ADDRESS)
    {
        for (i = 0; i < Mmu->gpuAddressRangeCount; i++)
        {
            if ((Address >= Mmu->gpuAddressRanges[i].start) &&
                (Address <= Mmu->gpuAddressRanges[i].end))
            {
                inFlatmapping = gcvTRUE;
                break;
            }
        }
    }

    *In = inFlatmapping;

    gcmkFOOTER_NO();
    return gcvSTATUS_OK;

OnError:
    gcmkFOOTER();
    return status;
}

gceSTATUS
gckMMU_SetupSRAM(
    IN gckMMU Mmu,
    IN gckHARDWARE Hardware,
    IN gckDEVICE Device
    )
{
    gctBOOL needMapInternalSRAM = gcvFALSE;
    gctPHYS_ADDR_T reservedBase = gcvINVALID_PHYSICAL_ADDRESS;
    gctUINT32 reservedSize = 0;
    gctUINT i = 0;
    gctUINT j = 0;
    gceSTATUS status;
    gckKERNEL kernel = Hardware->kernel;

    gcmkHEADER_ARG("Mmu=0x%x Hardware=0x%x", Mmu, Hardware);

    gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE);

    if (Hardware->mmuVersion == 0)
    {
        gcmkONERROR(gcvSTATUS_OK);
    }

    if (!Mmu->sRAMMapped)
    {
        gctUINT32 address = gcvINVALID_ADDRESS;
        gctUINT32 size    = 0;

        /* Map all the SRAMs in MMU table. */
        for (i = 0; i < gcvCORE_COUNT; i++)
        {
            for (j = gcvSRAM_INTERNAL0; j < gcvSRAM_INTER_COUNT; j++)
            {
                reservedBase = Device->sRAMBases[i][j];
                reservedSize = Device->sRAMSizes[i][j];

                Device->sRAMBaseAddresses[i][j] = 0;

                needMapInternalSRAM = reservedSize && (reservedBase != gcvINVALID_PHYSICAL_ADDRESS);

                /* Map the internal SRAM. */
                if (needMapInternalSRAM)
                {
                    if (Device->showSRAMMapInfo)
                    {
                        gcmkPRINT("Galcore Info: MMU mapped core%d SRAM base=0x%llx size=0x%x",
                            i,
                            reservedBase,
                            reservedSize
                            );
                    }

                    /*
                     * Default gpu virtual base = 0.
                     * It can be specified if not conflict with existing mapping.
                     */
                    gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(
                        Mmu->os,
                        reservedBase,
                        &reservedBase
                        ));

                    gcmkONERROR(_FillFlatMapping(
                        Mmu,
                        reservedBase,
                        reservedSize,
                        gcvTRUE,
                        gcvTRUE,
                        &Device->sRAMBaseAddresses[i][j]
                        ));

                    Device->sRAMBases[i][j] = reservedBase;
                }
                else if (reservedSize && reservedBase == gcvINVALID_PHYSICAL_ADDRESS)
                {
                    /*
                     * Reserve the internal SRAM range in first reserved MMU mtlb,
                     * when CPU physical base address is not specified, which means it is reserve usage.
                     */
                    Device->sRAMBaseAddresses[i][j] = (address == gcvINVALID_ADDRESS) ? gcdRESERVE_START :
                                                      address + gcmALIGN(size, gcdRESERVE_ALIGN);

                    Device->sRAMBases[i][j] = address = Device->sRAMBaseAddresses[i][j];

                    size = Device->sRAMSizes[i][j];

                    Device->sRAMPhysFaked[i][j] = gcvTRUE;
                }

#if gcdCAPTURE_ONLY_MODE
                Device->sRAMPhysFaked[i][j] = gcvTRUE;
#endif
            }
        }

        /* Map all the external SRAMs in MMU table. */
        for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++)
        {
            if (Device->extSRAMSizes[i] &&
               (Device->extSRAMBases[i] != gcvINVALID_PHYSICAL_ADDRESS))
            {

                Device->extSRAMBaseAddresses[i] = 0;

                gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(
                    Mmu->os,
                    Device->extSRAMBases[i],
                    &Device->extSRAMGPUBases[i]
                    ));

                gcmkONERROR(_FillFlatMapping(
                    Mmu,
                    Device->extSRAMGPUBases[i],
                    Device->extSRAMSizes[i],
                    gcvFALSE,
                    gcvTRUE,
                    &Device->extSRAMBaseAddresses[i]
                    ));

                Device->extSRAMGPUPhysNames[i] = gckKERNEL_AllocateNameFromPointer(kernel, Device->extSRAMPhysical[i]);
            }
        }
        Mmu->sRAMMapped = gcvTRUE;
    }

    /* Get per core SRAM hardware base address. */
    for (i = gcvSRAM_INTERNAL0; i < gcvSRAM_INTER_COUNT; i++)
    {
        if (Device->sRAMSizes[Hardware->core][i] &&
           (Device->sRAMBaseAddresses[Hardware->core][i]))
        {
            kernel->sRAMBaseAddresses[i] = Device->sRAMBaseAddresses[Hardware->core][i];
            kernel->sRAMSizes[i] = Hardware->options.sRAMSizes[i]
                                 = Device->sRAMSizes[Hardware->core][i];

            kernel->sRAMPhysFaked[i] = Device->sRAMPhysFaked[Hardware->core][i];
            Hardware->options.sRAMGPUVirtAddrs[i] = Device->sRAMBaseAddresses[Hardware->core][i];

            /* If the internal SRAM usage is reserve. */
            if (kernel->sRAMPhysFaked[i])
            {
                /* Use virtual address as the faked physical address which will never be accessed. */
                status = gckVIDMEM_Construct(
                    Mmu->os,
                    (gctPHYS_ADDR_T)kernel->sRAMBaseAddresses[i],
                    kernel->sRAMSizes[i],
                    64,
                    0,
                    &kernel->sRAMVidMem[i]
                    );

                if (gcmIS_ERROR(status))
                {
                    kernel->sRAMSizes[i] = 0;
                    kernel->sRAMVidMem[i] = gcvNULL;
                }
                else
                {
                    gcmkONERROR(gckOS_RequestReservedMemory(
                        Mmu->os,
                        (gctPHYS_ADDR_T)kernel->sRAMBaseAddresses[i],
                        kernel->sRAMSizes[i],
                        "Per core SRAM reserve usage region",
                        gcvTRUE,
                        &kernel->sRAMPhysical[i]
                        ));

                    kernel->sRAMVidMem[i]->physical = kernel->sRAMPhysical[i];
                }
            }

            if (Device->showSRAMMapInfo)
            {
                gcmkPRINT("Galcore Info: MMU mapped core %d SRAM[%d] hardware virtual address=0x%x size=0x%x",
                    Hardware->core,
                    i,
                    kernel->sRAMBaseAddresses[i],
                    kernel->sRAMSizes[i]
                    );
            }
        }
    }

    for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++)
    {
        if (Device->extSRAMSizes[i] &&
            (Device->extSRAMBases[i] != gcvINVALID_PHYSICAL_ADDRESS))
        {
            kernel->extSRAMBaseAddresses[i] = Device->extSRAMBaseAddresses[i];

            Hardware->options.extSRAMGPUVirtAddrs[i] = Device->extSRAMBaseAddresses[i];
            Hardware->options.extSRAMSizes[i]        = Device->extSRAMSizes[i];
            Hardware->options.extSRAMCPUPhysAddrs[i] = Device->extSRAMBases[i];
            Hardware->options.extSRAMGPUPhysAddrs[i] = Device->extSRAMGPUBases[i];
            Hardware->options.extSRAMGPUPhysNames[i] = Device->extSRAMGPUPhysNames[i];

            if (Device->showSRAMMapInfo)
            {
                gcmkPRINT("Galcore Info: MMU mapped external shared SRAM[%d] CPU view base=0x%llx GPU view base=0x%llx GPU virtual address=0x%x size=0x%x",
                    i,
                    Device->extSRAMBases[i],
                    Device->extSRAMGPUBases[i],
                    kernel->extSRAMBaseAddresses[i],
                    Device->extSRAMSizes[i]
                    );
            }
        }
    }

    gcsLIST_Add(&Hardware->mmuHead, &Mmu->hardwareList);

    gcmkFOOTER_NO();
    return gcvSTATUS_OK;

OnError:
    /* Return the error. */
    gcmkFOOTER();
    return status;
}


gceSTATUS
gckMMU_GetPageEntry(
    IN gckMMU Mmu,
    IN gcePAGE_TYPE PageType,
    IN gctUINT32 Address,
    IN gctUINT32_PTR *PageTable
    )
{
    gctUINT32_PTR pageTable;
    gctUINT32 index;
    gctUINT32 mtlbOffset, stlbOffset;
    gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, gcvFALSE);

    gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address);
    gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);

    gcmkASSERT(Mmu->hardware->mmuVersion > 0);

    mtlbOffset = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT;

    if (mtlbOffset >= area->mappingStart)
    {
        gctUINT32 stlbShift = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT;
        gctUINT32 stlbMask  = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_MASK : gcdMMU_STLB_4K_MASK;
        gctUINT32 stlbEntryNum = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM;

        stlbOffset = (Address & stlbMask) >> stlbShift;

        pageTable = area->stlbLogical;

        index = (mtlbOffset - area->mappingStart) * stlbEntryNum + stlbOffset;

        *PageTable = pageTable + index;
    }

    gcmkFOOTER_NO();
    return gcvSTATUS_OK;
}

gceSTATUS
gckMMU_GetAreaType(
    IN gckMMU Mmu,
    IN gctUINT32 GpuAddress,
    OUT gceAREA_TYPE *AreaType
    )
{
    gctUINT32 mtlbIndex;
    gctBOOL flatMapped;
    gceSTATUS status = gcvSTATUS_OK;
    gcsADDRESS_AREA_PTR area4K = &Mmu->dynamicArea4K;
#if gcdENABLE_GPU_1M_PAGE
    gcsADDRESS_AREA_PTR area1M = &Mmu->dynamicArea1M;
#endif

    mtlbIndex = _MtlbOffset(GpuAddress);

    gcmkONERROR(gckMMU_IsFlatMapped(Mmu, gcvINVALID_PHYSICAL_ADDRESS, GpuAddress, &flatMapped));

    if (flatMapped)
    {
        *AreaType = gcvAREA_TYPE_FLATMAP;
    }
#if gcdENABLE_GPU_1M_PAGE
    else if (mtlbIndex >= area1M->mappingStart && mtlbIndex <= area1M->mappingEnd)
    {
        *AreaType = gcvAREA_TYPE_1M;
    }
#endif
    else if (mtlbIndex >= area4K->mappingStart && mtlbIndex <= area4K->mappingEnd)
    {
        *AreaType = gcvAREA_TYPE_4K;
    }
    else
    {
        *AreaType = gcvAREA_TYPE_UNKNOWN;
    }

OnError:
    return status;
}