Google Git
Sign in
coral / imx-gpu-viv-ko / refs/heads/master / . / hal / kernel / gc_hal_kernel_video_memory.c
blob: 7e3b93cfc3f870d6c889e4efcda88776566ab463 [file] [log] [blame]
/****************************************************************************
*
* The MIT License (MIT)
*
* Copyright (c) 2014 - 2020 Vivante Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
*****************************************************************************
*
* The GPL License (GPL)
*
* Copyright (C) 2014 - 2020 Vivante Corporation
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*****************************************************************************
*
* Note: This software is released under dual MIT and GPL licenses. A
* recipient may use this file under the terms of either the MIT license or
* GPL License. If you wish to use only one license not the other, you can
* indicate your decision by deleting one of the above license notices in your
* version of this file.
*
*****************************************************************************/
#include "gc_hal_kernel_precomp.h"
#define _GC_OBJ_ZONE gcvZONE_VIDMEM
/******************************************************************************\
******************************* Private Functions ******************************
\******************************************************************************/
/*******************************************************************************
**
** _Split
**
** Split a node on the required byte boundary.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to the node to split.
**
** gctSIZE_T Bytes
** Number of bytes to keep in the node.
**
** OUTPUT:
**
** Nothing.
**
** RETURNS:
**
** gctBOOL
** gcvTRUE if the node was split successfully, or gcvFALSE if there is an
** error.
**
*/
static gctBOOL
_Split(
IN gckOS Os,
IN gcuVIDMEM_NODE_PTR Node,
IN gctSIZE_T Bytes
)
{
gcuVIDMEM_NODE_PTR node;
gctPOINTER pointer = gcvNULL;
/* Make sure the byte boundary makes sense. */
if ((Bytes <= 0) || (Bytes > Node->VidMem.bytes))
{
return gcvFALSE;
}
/* Allocate a new gcuVIDMEM_NODE object. */
if (gcmIS_ERROR(gckOS_Allocate(Os,
gcmSIZEOF(gcuVIDMEM_NODE),
&pointer)))
{
/* Error. */
return gcvFALSE;
}
node = pointer;
/* Initialize gcuVIDMEM_NODE structure. */
node->VidMem.offset = Node->VidMem.offset + Bytes;
node->VidMem.bytes = Node->VidMem.bytes - Bytes;
node->VidMem.alignment = 0;
node->VidMem.locked = 0;
node->VidMem.parent = Node->VidMem.parent;
node->VidMem.pool = Node->VidMem.pool;
node->VidMem.processID = 0;
node->VidMem.logical = gcvNULL;
node->VidMem.kvaddr = gcvNULL;
/* Insert node behind specified node. */
node->VidMem.next = Node->VidMem.next;
node->VidMem.prev = Node;
Node->VidMem.next = node->VidMem.next->VidMem.prev = node;
/* Insert free node behind specified node. */
node->VidMem.nextFree = Node->VidMem.nextFree;
node->VidMem.prevFree = Node;
Node->VidMem.nextFree = node->VidMem.nextFree->VidMem.prevFree = node;
/* Adjust size of specified node. */
Node->VidMem.bytes = Bytes;
/* Success. */
return gcvTRUE;
}
/*******************************************************************************
**
** _Merge
**
** Merge two adjacent nodes together.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to the first of the two nodes to merge.
**
** OUTPUT:
**
** Nothing.
**
*/
static gceSTATUS
_Merge(
IN gckOS Os,
IN gcuVIDMEM_NODE_PTR Node
)
{
gcuVIDMEM_NODE_PTR node;
gceSTATUS status;
/* Save pointer to next node. */
node = Node->VidMem.next;
/* This is a good time to make sure the heap is not corrupted. */
if (Node->VidMem.offset + Node->VidMem.bytes != node->VidMem.offset)
{
/* Corrupted heap. */
gcmkASSERT(
Node->VidMem.offset + Node->VidMem.bytes == node->VidMem.offset);
return gcvSTATUS_HEAP_CORRUPTED;
}
/* Adjust byte count. */
Node->VidMem.bytes += node->VidMem.bytes;
/* Unlink next node from linked list. */
Node->VidMem.next = node->VidMem.next;
Node->VidMem.nextFree = node->VidMem.nextFree;
Node->VidMem.next->VidMem.prev =
Node->VidMem.nextFree->VidMem.prevFree = Node;
/* Free next node. */
status = gcmkOS_SAFE_FREE(Os, node);
return status;
}
/******************************************************************************\
******************************* gckVIDMEM API Code ******************************
\******************************************************************************/
/*******************************************************************************
**
** gckVIDMEM_Construct
**
** Construct a new gckVIDMEM object.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR_T PhysicalBase
** Base physical address for the video memory heap.
**
** gctSIZE_T Bytes
** Number of bytes in the video memory heap.
**
** gctSIZE_T Threshold
** Minimum number of bytes beyond am allocation before the node is
** split. Can be used as a minimum alignment requirement.
**
** gctSIZE_T BankSize
** Number of bytes per physical memory bank. Used by bank
** optimization.
**
** OUTPUT:
**
** gckVIDMEM * Memory
** Pointer to a variable that will hold the pointer to the gckVIDMEM
** object.
*/
gceSTATUS
gckVIDMEM_Construct(
IN gckOS Os,
IN gctPHYS_ADDR_T PhysicalBase,
IN gctSIZE_T Bytes,
IN gctSIZE_T Threshold,
IN gctSIZE_T BankSize,
OUT gckVIDMEM * Memory
)
{
gckVIDMEM memory = gcvNULL;
gceSTATUS status;
gcuVIDMEM_NODE_PTR node;
gctINT i, banks = 0;
gctPOINTER pointer = gcvNULL;
gctUINT32 heapBytes;
gctUINT32 bankSize;
gctUINT32 base = 0;
gcmkHEADER_ARG("Os=0x%x PhysicalBase=%12llx Bytes=%lu Threshold=%lu "
"BankSize=%lu",
Os, PhysicalBase, Bytes, Threshold, BankSize);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Memory != gcvNULL);
gcmkSAFECASTSIZET(heapBytes, Bytes);
gcmkSAFECASTSIZET(bankSize, BankSize);
/* Allocate the gckVIDMEM object. */
gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(struct _gckVIDMEM), &pointer));
gckOS_ZeroMemory(pointer, gcmSIZEOF(struct _gckVIDMEM));
memory = pointer;
/* Initialize the gckVIDMEM object. */
memory->object.type = gcvOBJ_VIDMEM;
memory->os = Os;
/* Set video memory heap information. */
memory->physicalBase = PhysicalBase;
memory->bytes = heapBytes;
memory->freeBytes = heapBytes;
memory->minFreeBytes = heapBytes;
memory->capability = ~0u;
memory->threshold = Threshold;
memory->mutex = gcvNULL;
/* Walk all possible banks. */
for (i = 0; i < gcmCOUNTOF(memory->sentinel); ++i)
{
gctUINT32 bytes;
if (BankSize == 0)
{
/* Use all bytes for the first bank. */
bytes = heapBytes;
}
else
{
/* Compute number of bytes for this bank. */
bytes = gcmALIGN(base + 1, bankSize) - base;
if (bytes > heapBytes)
{
/* Make sure we don't exceed the total number of bytes. */
bytes = heapBytes;
}
}
if (bytes == 0)
{
/* Mark heap is not used. */
memory->sentinel[i].VidMem.next =
memory->sentinel[i].VidMem.prev =
memory->sentinel[i].VidMem.nextFree =
memory->sentinel[i].VidMem.prevFree = gcvNULL;
continue;
}
/* Allocate one gcuVIDMEM_NODE union. */
gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer));
node = pointer;
/* Initialize gcuVIDMEM_NODE union. */
node->VidMem.parent = memory;
node->VidMem.next =
node->VidMem.prev =
node->VidMem.nextFree =
node->VidMem.prevFree = &memory->sentinel[i];
node->VidMem.offset = base;
node->VidMem.bytes = bytes;
node->VidMem.alignment = 0;
node->VidMem.pool = gcvPOOL_UNKNOWN;
node->VidMem.locked = 0;
node->VidMem.processID = 0;
node->VidMem.logical = gcvNULL;
node->VidMem.kvaddr = gcvNULL;
/* Initialize the linked list of nodes. */
memory->sentinel[i].VidMem.next =
memory->sentinel[i].VidMem.prev =
memory->sentinel[i].VidMem.nextFree =
memory->sentinel[i].VidMem.prevFree = node;
/* Mark sentinel. */
memory->sentinel[i].VidMem.bytes = 0;
/* Adjust address for next bank. */
base += bytes;
heapBytes -= bytes;
banks ++;
}
/* Assign all the bank mappings. */
memory->mapping[gcvVIDMEM_TYPE_COLOR_BUFFER] = banks - 1;
memory->mapping[gcvVIDMEM_TYPE_BITMAP] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_DEPTH_BUFFER] = banks - 1;
memory->mapping[gcvVIDMEM_TYPE_HZ_BUFFER] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_TEXTURE] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_VERTEX_BUFFER] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_INDEX_BUFFER] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_TILE_STATUS] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_COMMAND] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_GENERIC] = 0;
memory->mapping[gcvVIDMEM_TYPE_ICACHE] = 0;
memory->mapping[gcvVIDMEM_TYPE_TXDESC] = 0;
memory->mapping[gcvVIDMEM_TYPE_FENCE] = 0;
memory->mapping[gcvVIDMEM_TYPE_TFBHEADER] = 0;
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] INDEX: bank %d",
memory->mapping[gcvVIDMEM_TYPE_INDEX_BUFFER]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] VERTEX: bank %d",
memory->mapping[gcvVIDMEM_TYPE_VERTEX_BUFFER]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] TEXTURE: bank %d",
memory->mapping[gcvVIDMEM_TYPE_TEXTURE]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] RENDER_TARGET: bank %d",
memory->mapping[gcvVIDMEM_TYPE_COLOR_BUFFER]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] DEPTH: bank %d",
memory->mapping[gcvVIDMEM_TYPE_DEPTH_BUFFER]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] TILE_STATUS: bank %d",
memory->mapping[gcvVIDMEM_TYPE_TILE_STATUS]);
/* Allocate the mutex. */
gcmkONERROR(gckOS_CreateMutex(Os, &memory->mutex));
/* Return pointer to the gckVIDMEM object. */
*Memory = memory;
/* Success. */
gcmkFOOTER_ARG("*Memory=0x%x", *Memory);
return gcvSTATUS_OK;
OnError:
/* Roll back. */
if (memory != gcvNULL)
{
if (memory->mutex != gcvNULL)
{
/* Delete the mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Os, memory->mutex));
}
for (i = 0; i < banks; ++i)
{
/* Free the heap. */
gcmkASSERT(memory->sentinel[i].VidMem.next != gcvNULL);
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory->sentinel[i].VidMem.next));
}
/* Free the object. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_Destroy
**
** Destroy an gckVIDMEM object.
**
** INPUT:
**
** gckVIDMEM Memory
** Pointer to an gckVIDMEM object to destroy.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckVIDMEM_Destroy(
IN gckVIDMEM Memory
)
{
gcuVIDMEM_NODE_PTR node, next;
gctINT i;
gcmkHEADER_ARG("Memory=0x%x", Memory);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM);
/* Walk all sentinels. */
for (i = 0; i < gcmCOUNTOF(Memory->sentinel); ++i)
{
/* Bail out of the heap is not used. */
if (Memory->sentinel[i].VidMem.next == gcvNULL)
{
break;
}
/* Walk all the nodes until we reach the sentinel. */
for (node = Memory->sentinel[i].VidMem.next;
node->VidMem.bytes != 0;
node = next)
{
/* Save pointer to the next node. */
next = node->VidMem.next;
/* Free the node. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, node));
}
}
/* Free the mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Memory->os, Memory->mutex));
/* Mark the object as unknown. */
Memory->object.type = gcvOBJ_UNKNOWN;
/* Free the gckVIDMEM object. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, Memory));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
#if gcdENABLE_BANK_ALIGNMENT
#if !gcdBANK_BIT_START
#error gcdBANK_BIT_START not defined.
#endif
#if !gcdBANK_BIT_END
#error gcdBANK_BIT_END not defined.
#endif
/*******************************************************************************
** _GetSurfaceBankAlignment
**
** Return the required offset alignment required to the make BaseAddress
** aligned properly.
**
** INPUT:
**
** gckOS Os
** Pointer to gcoOS object.
**
** gceVIDMEM_TYPE Type
** Type of allocation.
**
** gctUINT32 BaseAddress
** Base address of current video memory node.
**
** OUTPUT:
**
** gctUINT32_PTR AlignmentOffset
** Pointer to a variable that will hold the number of bytes to skip in
** the current video memory node in order to make the alignment bank
** aligned.
*/
static gceSTATUS
_GetSurfaceBankAlignment(
IN gckKERNEL Kernel,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 BaseAddress,
OUT gctUINT32_PTR AlignmentOffset
)
{
gctUINT32 bank;
/* To retrieve the bank. */
static const gctUINT32 bankMask = (0xFFFFFFFF << gcdBANK_BIT_START)
^ (0xFFFFFFFF << (gcdBANK_BIT_END + 1));
/* To retrieve the bank and all the lower bytes. */
static const gctUINT32 byteMask = ~(0xFFFFFFFF << (gcdBANK_BIT_END + 1));
gcmkHEADER_ARG("Type=%d BaseAddress=0x%x ", Type, BaseAddress);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(AlignmentOffset != gcvNULL);
switch (Type)
{
case gcvVIDMEM_TYPE_COLOR_BUFFER:
bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START);
/* Align to the first bank. */
*AlignmentOffset = (bank == 0) ?
0 :
((1 << (gcdBANK_BIT_END + 1)) + 0) - (BaseAddress & byteMask);
break;
case gcvVIDMEM_TYPE_DEPTH_BUFFER:
bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START);
/* Align to the third bank. */
*AlignmentOffset = (bank == 2) ?
0 :
((1 << (gcdBANK_BIT_END + 1)) + (2 << gcdBANK_BIT_START)) - (BaseAddress & byteMask);
/* Minimum 256 byte alignment needed for fast_msaa. */
if ((gcdBANK_CHANNEL_BIT > 7) ||
((gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_FAST_MSAA) != gcvSTATUS_TRUE) &&
(gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_SMALL_MSAA) != gcvSTATUS_TRUE)))
{
/* Add a channel offset at the channel bit. */
*AlignmentOffset += (1 << gcdBANK_CHANNEL_BIT);
}
break;
default:
/* no alignment needed. */
*AlignmentOffset = 0;
}
/* Return the status. */
gcmkFOOTER_ARG("*AlignmentOffset=%u", *AlignmentOffset);
return gcvSTATUS_OK;
}
#endif
static gcuVIDMEM_NODE_PTR
_FindNode(
IN gckKERNEL Kernel,
IN gckVIDMEM Memory,
IN gctINT Bank,
IN gctSIZE_T Bytes,
IN gceVIDMEM_TYPE Type,
IN OUT gctUINT32_PTR Alignment
)
{
gcuVIDMEM_NODE_PTR node;
gctUINT32 alignment;
#if gcdENABLE_BANK_ALIGNMENT
gctUINT32 bankAlignment;
gceSTATUS status;
#endif
if (Memory->sentinel[Bank].VidMem.nextFree == gcvNULL)
{
/* No free nodes left. */
return gcvNULL;
}
#if gcdENABLE_BANK_ALIGNMENT
/* Walk all free nodes until we have one that is big enough or we have
** reached the sentinel. */
for (node = Memory->sentinel[Bank].VidMem.nextFree;
node->VidMem.bytes != 0;
node = node->VidMem.nextFree)
{
if (node->VidMem.bytes < Bytes)
{
continue;
}
gcmkONERROR(_GetSurfaceBankAlignment(
Kernel,
Type,
(gctUINT32)(node->VidMem.parent->physicalBase + node->VidMem.offset),
&bankAlignment));
bankAlignment = gcmALIGN(bankAlignment, *Alignment);
/* Compute number of bytes to skip for alignment. */
alignment = (*Alignment == 0)
? 0
: (*Alignment - (node->VidMem.offset % *Alignment));
if (alignment == *Alignment)
{
/* Node is already aligned. */
alignment = 0;
}
if (node->VidMem.bytes >= Bytes + alignment + bankAlignment)
{
/* This node is big enough. */
*Alignment = alignment + bankAlignment;
return node;
}
}
#endif
/* Walk all free nodes until we have one that is big enough or we have
reached the sentinel. */
for (node = Memory->sentinel[Bank].VidMem.nextFree;
node->VidMem.bytes != 0;
node = node->VidMem.nextFree)
{
gctUINT offset;
gctINT modulo;
gcmkSAFECASTSIZET(offset, node->VidMem.offset);
modulo = gckMATH_ModuloInt(offset, *Alignment);
/* Compute number of bytes to skip for alignment. */
alignment = (*Alignment == 0) ? 0 : (*Alignment - modulo);
if (alignment == *Alignment)
{
/* Node is already aligned. */
alignment = 0;
}
if (node->VidMem.bytes >= Bytes + alignment)
{
/* This node is big enough. */
*Alignment = alignment;
return node;
}
}
#if gcdENABLE_BANK_ALIGNMENT
OnError:
#endif
/* Not enough memory. */
return gcvNULL;
}
/*******************************************************************************
**
** gckVIDMEM_AllocateLinear
**
** Allocate linear memory from the gckVIDMEM object.
**
** INPUT:
**
** gckVIDMEM Memory
** Pointer to an gckVIDMEM object.
**
** gctSIZE_T Bytes
** Number of bytes to allocate.
**
** gctUINT32 Alignment
** Byte alignment for allocation.
**
** gceVIDMEM_TYPE Type
** Type of surface to allocate (use by bank optimization).
**
** gctUINT32 Flag
** Flag of allocatetion.
**
** gctBOOL Specified
** If user must use this pool, it should set Specified to gcvTRUE,
** otherwise allocator may reserve some memory for other usage, such
** as small block size allocation request.
**
** OUTPUT:
**
** gcuVIDMEM_NODE_PTR * Node
** Pointer to a variable that will hold the allocated memory node.
*/
static gceSTATUS
gckVIDMEM_AllocateLinear(
IN gckKERNEL Kernel,
IN gckVIDMEM Memory,
IN gctSIZE_T Bytes,
IN gctUINT32 Alignment,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
IN gctBOOL Specified,
OUT gcuVIDMEM_NODE_PTR * Node
)
{
gceSTATUS status;
gcuVIDMEM_NODE_PTR node;
gctUINT32 alignment;
gctINT bank, i;
gctBOOL acquired = gcvFALSE;
gctUINT64 mappingInOne = 1;
gcmkHEADER_ARG("Memory=0x%x Bytes=%lu Alignment=%u Type=%d",
Memory, Bytes, Alignment, Type);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Node != gcvNULL);
gcmkVERIFY_ARGUMENT(Type < gcvVIDMEM_TYPE_COUNT);
/* Acquire the mutex. */
gcmkONERROR(gckOS_AcquireMutex(Memory->os, Memory->mutex, gcvINFINITE));
acquired = gcvTRUE;
if (Bytes > Memory->freeBytes)
{
/* Not enough memory. */
status = gcvSTATUS_OUT_OF_MEMORY;
goto OnError;
}
#if gcdSMALL_BLOCK_SIZE
if ((Memory->freeBytes < (Memory->bytes/gcdRATIO_FOR_SMALL_MEMORY))
&& (Bytes >= gcdSMALL_BLOCK_SIZE)
&& (Specified == gcvFALSE)
)
{
/* The left memory is for small memory.*/
status = gcvSTATUS_OUT_OF_MEMORY;
goto OnError;
}
#endif
/* Find the default bank for this surface type. */
gcmkASSERT((gctINT) Type < gcmCOUNTOF(Memory->mapping));
bank = Memory->mapping[Type];
alignment = Alignment;
/* Find a free node in the default bank. */
node = _FindNode(Kernel, Memory, bank, Bytes, Type, &alignment);
/* Out of memory? */
if (node == gcvNULL)
{
/* Walk all lower banks. */
for (i = bank - 1; i >= 0; --i)
{
/* Find a free node inside the current bank. */
node = _FindNode(Kernel, Memory, i, Bytes, Type, &alignment);
if (node != gcvNULL)
{
break;
}
}
}
if (node == gcvNULL)
{
/* Walk all upper banks. */
for (i = bank + 1; i < gcmCOUNTOF(Memory->sentinel); ++i)
{
if (Memory->sentinel[i].VidMem.nextFree == gcvNULL)
{
/* Abort when we reach unused banks. */
break;
}
/* Find a free node inside the current bank. */
node = _FindNode(Kernel, Memory, i, Bytes, Type, &alignment);
if (node != gcvNULL)
{
break;
}
}
}
if (node == gcvNULL)
{
/* Out of memory. */
status = gcvSTATUS_OUT_OF_MEMORY;
goto OnError;
}
/* Do we have an alignment? */
if (alignment > 0)
{
/* Split the node so it is aligned. */
if (_Split(Memory->os, node, alignment))
{
/* Successful split, move to aligned node. */
node = node->VidMem.next;
/* Remove alignment. */
alignment = 0;
}
}
/* Do we have enough memory after the allocation to split it? */
if (node->VidMem.bytes - Bytes > Memory->threshold)
{
/* Adjust the node size. */
_Split(Memory->os, node, Bytes);
}
/* Remove the node from the free list. */
node->VidMem.prevFree->VidMem.nextFree = node->VidMem.nextFree;
node->VidMem.nextFree->VidMem.prevFree = node->VidMem.prevFree;
node->VidMem.nextFree =
node->VidMem.prevFree = gcvNULL;
/* Fill in the information. */
node->VidMem.alignment = alignment;
node->VidMem.parent = Memory;
node->VidMem.logical = gcvNULL;
gcmkONERROR(gckOS_GetProcessID(&node->VidMem.processID));
/* Adjust the number of free bytes. */
Memory->freeBytes -= node->VidMem.bytes;
if (Memory->freeBytes < Memory->minFreeBytes)
{
Memory->minFreeBytes = Memory->freeBytes;
}
gckOS_QueryOption(Memory->os, "allMapInOne", &mappingInOne);
if (!mappingInOne)
{
gcmkONERROR(gckOS_RequestReservedMemory(
Memory->os,
Memory->physicalBase + node->VidMem.offset,
node->VidMem.bytes,
"gal reserved memory",
gcvTRUE,
&node->VidMem.physical
));
}
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex));
/* Return the pointer to the node. */
*Node = node;
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Allocated %u bytes @ 0x%x [0x%08X]",
node->VidMem.bytes, node, node->VidMem.offset);
/* Success. */
gcmkFOOTER_ARG("*Node=0x%x", *Node);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_AllocateVirtual
**
** Construct a new gcuVIDMEM_NODE union for virtual memory.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctSIZE_T Bytes
** Number of byte to allocate.
**
** OUTPUT:
**
** gcuVIDMEM_NODE_PTR * Node
** Pointer to a variable that receives the gcuVIDMEM_NODE union pointer.
*/
static gceSTATUS
gckVIDMEM_AllocateVirtual(
IN gckKERNEL Kernel,
IN gctUINT32 Flag,
IN gctSIZE_T Bytes,
OUT gcuVIDMEM_NODE_PTR * Node
)
{
gckOS os;
gceSTATUS status;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gctPOINTER pointer = gcvNULL;
gctINT i;
gcmkHEADER_ARG("Kernel=0x%x Flag=%x Bytes=%lu", Kernel, Flag, Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Node != gcvNULL);
/* Extract the gckOS object pointer. */
os = Kernel->os;
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
/* Allocate an gcuVIDMEM_NODE union. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer));
node = pointer;
/* Initialize gcuVIDMEM_NODE union for virtual memory. */
node->Virtual.kernel = Kernel;
node->Virtual.contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS;
node->Virtual.logical = gcvNULL;
node->Virtual.kvaddr = gcvNULL;
node->Virtual.bytes = Bytes;
node->Virtual.secure = (Flag & gcvALLOC_FLAG_SECURITY) != 0;
node->Virtual.onFault = (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT) != 0;
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
node->Virtual.lockeds[i] = 0;
node->Virtual.pageTables[i] = gcvNULL;
}
/* Allocate the virtual memory. */
gcmkONERROR(
gckOS_AllocatePagedMemory(os,
Flag,
&node->Virtual.bytes,
&node->Virtual.gid,
&node->Virtual.physical));
/* Calculate required GPU page (4096) count. */
/* Assume start address is 4096 aligned. */
node->Virtual.pageCount = (gctSIZE_T)(((gctUINT64)node->Virtual.bytes + (4096 - 1)) >> 12);
/* Return pointer to the gcuVIDMEM_NODE union. */
*Node = node;
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Created virtual node 0x%x for %u bytes @ 0x%x",
node, Bytes, node->Virtual.physical);
/* Success. */
gcmkFOOTER_ARG("*Node=0x%x", *Node);
return gcvSTATUS_OK;
OnError:
/* Roll back. */
if (node != gcvNULL)
{
/* Free the structure. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
static gceSTATUS
_AddToBlockList(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
VidMemBlock->next = Kernel->vidMemBlock;
Kernel->vidMemBlock = VidMemBlock;
return gcvSTATUS_OK;
}
static gceSTATUS
_RemoveFromBlockList(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
gckVIDMEM_BLOCK vidMemBlock;
gckVIDMEM_BLOCK previous = gcvNULL;
gceHARDWARE_TYPE hwType;
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
for (vidMemBlock = Kernel->vidMemBlock; vidMemBlock != gcvNULL; vidMemBlock = vidMemBlock->next)
{
if (vidMemBlock->addresses[hwType] == VidMemBlock->addresses[hwType])
{
if (previous)
{
previous->next = vidMemBlock->next;
}
else
{
Kernel->vidMemBlock = vidMemBlock->next;
}
vidMemBlock->next = gcvNULL;
break;
}
previous = vidMemBlock;
}
return gcvSTATUS_OK;
}
static gckVIDMEM_BLOCK
_FindFreeBlock(
IN gckKERNEL Kernel,
IN gctSIZE_T Bytes
)
{
gckVIDMEM_BLOCK vidMemBlock;
for (vidMemBlock = Kernel->vidMemBlock; vidMemBlock != gcvNULL; vidMemBlock = vidMemBlock->next)
{
if (vidMemBlock->freeBytes >= Bytes)
{
/* Found the block */
break;
}
}
return vidMemBlock;
}
static gceSTATUS
_SplitVirtualChunk(
IN gckOS Os,
IN gcuVIDMEM_NODE_PTR Node,
IN gctSIZE_T Bytes
)
{
gceSTATUS status = gcvSTATUS_OK;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gctPOINTER pointer;
gctINT i;
if ((Bytes <= 0) || (Bytes > Node->VirtualChunk.bytes))
{
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
/* Allocate a new gcuVIDMEM_NODE object. */
gcmkONERROR(gckOS_Allocate(Os,
gcmSIZEOF(gcuVIDMEM_NODE),
&pointer));
node = pointer;
/* Intialize the new gcuVIDMEM_NODE. */
node->VirtualChunk.offset = Node->VirtualChunk.offset + Bytes;
node->VirtualChunk.bytes = Node->VirtualChunk.bytes - Bytes;
node->VirtualChunk.parent = Node->VirtualChunk.parent;
node->VirtualChunk.kernel = Node->VirtualChunk.kernel;
node->VirtualChunk.kvaddr = gcvNULL;
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
node->VirtualChunk.lockeds[i] = 0;
}
/* Insert chunk behind specified chunk. */
node->VirtualChunk.next = Node->VirtualChunk.next;
node->VirtualChunk.prev = Node;
Node->VirtualChunk.next = node->VirtualChunk.next->VirtualChunk.prev = node;
/* Insert free chunk behind specified chunk. */
node->VirtualChunk.nextFree = Node->VirtualChunk.nextFree;
node->VirtualChunk.prevFree = Node;
Node->VirtualChunk.nextFree = node->VirtualChunk.nextFree->VirtualChunk.prevFree = node;
/* Adjust size of specified chunk. */
Node->VirtualChunk.bytes = Bytes;
OnError:
return status;
}
static gceSTATUS
_MergeVirtualChunk(
IN gckOS Os,
IN gcuVIDMEM_NODE_PTR Node
)
{
gcuVIDMEM_NODE_PTR node;
gceSTATUS status = gcvSTATUS_OK;
node = Node->VirtualChunk.next;
if (Node->VirtualChunk.offset + Node->VirtualChunk.bytes != node->VirtualChunk.offset)
{
/* Corrupted heap. */
gcmkASSERT(
Node->VirtualChunk.offset + Node->VirtualChunk.bytes == node->VirtualChunk.offset);
return gcvSTATUS_HEAP_CORRUPTED;
}
/* Merge. */
Node->VirtualChunk.bytes += node->VirtualChunk.bytes;
/* Unlink next node from linked list. */
Node->VirtualChunk.next = node->VirtualChunk.next;
Node->VirtualChunk.nextFree = node->VirtualChunk.nextFree;
Node->VirtualChunk.next->VirtualChunk.prev =
Node->VirtualChunk.nextFree->VirtualChunk.prevFree = Node;
/* Free the next node. */
status = gcmkOS_SAFE_FREE(Os, node);
return status;
}
static gctBOOL
_IsVidMemBlockFree(
IN gckVIDMEM_BLOCK VidMemBlock
)
{
return (VidMemBlock->freeBytes == VidMemBlock->bytes);
}
static gcuVIDMEM_NODE_PTR
_FindVirtualChunkNode(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock,
IN gctSIZE_T Bytes
)
{
gcuVIDMEM_NODE_PTR node;
if (VidMemBlock->node.VirtualChunk.nextFree == gcvNULL)
{
/* No free chunk left. */
return gcvNULL;
}
for (node = VidMemBlock->node.VirtualChunk.nextFree;
node->VirtualChunk.bytes != 0;
node = node->VirtualChunk.nextFree)
{
if (node->VirtualChunk.bytes >= Bytes)
{
/* Got it. */
return node;
}
}
return gcvNULL;
}
/*******************************************************************************
**
** _ConvertPhysical
**
** Convert CPU physical to GPU address for video node.
**
** INPUT:
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to a gcuVIDMEM_NODE union.
**
** gceCORE Core
** Id of current GPU.
**
** gctPHYS_ADDR_T PhysicalAddress
** CPU physical address
**
** OUTPUT:
** gctUINT32 * Address
** A pointer hold the GPU address.
*/
static gceSTATUS
_ConvertPhysical(
IN gckKERNEL Kernel,
IN gceCORE Core,
IN gcuVIDMEM_NODE_PTR Node,
IN gckVIDMEM_BLOCK VidMemBlock,
IN gctPHYS_ADDR_T PhysicalAddress,
OUT gctUINT32 * Address
)
{
gceSTATUS status;
gctUINT64 physical = 0;
gcmkHEADER_ARG("Node=0x%X", Node);
if ((Node && !Node->Virtual.contiguous) ||
(VidMemBlock && !VidMemBlock->contiguous))
{
/* non-contiguous, mapping is required. */
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
if ((Node && Node->Virtual.secure) ||
(VidMemBlock && VidMemBlock->secure))
{
/* Secure, mapping is forced. */
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
/* Convert to GPU physical address. */
gckOS_CPUPhysicalToGPUPhysical(Kernel->os, PhysicalAddress, &physical);
if ((physical > gcvMAXUINT32) ||
(Node && (physical + Node->Virtual.bytes - 1 > gcvMAXUINT32)) ||
(VidMemBlock && (physical + VidMemBlock->bytes - 1 > gcvMAXUINT32)))
{
/* Above 4G (32bit), mapping is required currently. */
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
if (!gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_MMU))
{
if (physical < Kernel->hardware->baseAddress)
{
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
/* Subtract baseAddress to get a GPU address used for programming. */
physical -= Kernel->hardware->baseAddress;
/* 2G upper is virtual space, better to move to gckHARDWARE section. */
if (physical + Node->Virtual.bytes > 0x80000000)
{
/* End is above 2G, ie virtual space. */
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
*Address = (gctUINT32)physical;
gcmkFOOTER_ARG("*Address=0x%X", *Address);
return gcvSTATUS_OK;
}
else
{
gctBOOL flatMapped;
gcmkONERROR(gckMMU_IsFlatMapped(Kernel->mmu, physical, gcvINVALID_ADDRESS, &flatMapped));
if (!flatMapped)
{
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
*Address = (gctUINT32)physical;
gcmkFOOTER_ARG("*Address=0x%X", *Address);
return gcvSTATUS_OK;
}
OnError:
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_MapVidMemBlock(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
gceSTATUS status;
gckOS os = Kernel->os;
gctPHYS_ADDR_T physAddr;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p", Kernel, VidMemBlock);
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
gcmkASSERT(VidMemBlock->pageCount > 0);
gcmkONERROR(
gckOS_GetPhysicalFromHandle(os,
VidMemBlock->physical,
0,
&physAddr));
status = _ConvertPhysical(Kernel,
Kernel->core,
gcvNULL,
VidMemBlock,
physAddr,
&VidMemBlock->addresses[hwType]);
if (gcmIS_ERROR(status))
{
/* Allocate pages inside the MMU. */
gcmkONERROR(
gckMMU_AllocatePagesEx(Kernel->mmu,
VidMemBlock->pageCount,
VidMemBlock->type,
gcvPAGE_TYPE_1M,
VidMemBlock->secure,
&VidMemBlock->pageTables[hwType],
&VidMemBlock->addresses[hwType]));
if (VidMemBlock->onFault != gcvTRUE)
{
/* Map the pages. */
gcmkONERROR(
gckOS_Map1MPages(os,
Kernel->core,
VidMemBlock->physical,
VidMemBlock->pageCount,
VidMemBlock->addresses[hwType],
VidMemBlock->pageTables[hwType],
gcvTRUE,
VidMemBlock->type));
}
gcmkONERROR(gckMMU_Flush(Kernel->mmu, VidMemBlock->type));
/* Calculate the GPU virtual address. */
VidMemBlock->addresses[hwType] |= (gctUINT32) (physAddr & ((1 << 20) - 1));
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Mapped video memory block 0x%x to 0x%08X",
VidMemBlock,
VidMemBlock->addresses[hwType]);
return gcvSTATUS_OK;
OnError:
if (VidMemBlock->pageTables[hwType] != gcvNULL)
{
/* Free the pages from the MMU. */
gcmkVERIFY_OK(
gckMMU_FreePages(Kernel->mmu,
VidMemBlock->secure,
gcvPAGE_TYPE_1M,
VidMemBlock->addresses[hwType],
VidMemBlock->pageTables[hwType],
VidMemBlock->pageCount));
VidMemBlock->pageTables[hwType] = gcvNULL;
}
gcmkFOOTER();
return status;
}
static gceSTATUS
_UnmapVidMemBlock(
IN gckMMU Mmu,
IN gceHARDWARE_TYPE HwType,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
gceSTATUS status;
gcmkHEADER_ARG("Mmu=%p VidMemBlock=%p", Mmu, VidMemBlock);
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
if (VidMemBlock->pageTables[HwType] != gcvNULL)
{
/* Free the pages from the MMU. */
gcmkONERROR(
gckMMU_FreePages(Mmu,
VidMemBlock->secure,
gcvPAGE_TYPE_1M,
VidMemBlock->addresses[HwType],
VidMemBlock->pageTables[HwType],
VidMemBlock->pageCount));
VidMemBlock->pageTables[HwType] = gcvNULL;
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_BLOCK_Construct(
IN gckKERNEL Kernel,
IN gctSIZE_T BlockSize,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
OUT gckVIDMEM_BLOCK * VidMemBlock
)
{
gceSTATUS status;
gckVIDMEM_BLOCK vidMemBlock = gcvNULL;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gckOS os = Kernel->os;
gctPOINTER pointer;
gctINT i;
gcmkHEADER_ARG("Kernel=0x%x BlockSize=%lu Type=%x Flag=%x", Kernel, BlockSize, Type);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(BlockSize > 0);
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
/* Allocate an gckVIDMEM_BLOCK object. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_BLOCK), &pointer));
vidMemBlock = pointer;
/* Initialize the gckVIDMEM_BLOCK object. */
vidMemBlock->object.type = gcvOBJ_VIDMEM_BLOCK;
vidMemBlock->os = os;
vidMemBlock->bytes = BlockSize;
vidMemBlock->freeBytes = BlockSize;
/* 1M page count. */
vidMemBlock->pageCount = (gctUINT32)(BlockSize >> 20);
vidMemBlock->type = Type;
vidMemBlock->contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS;
vidMemBlock->secure = (Flag & gcvALLOC_FLAG_SECURITY) != 0;
vidMemBlock->onFault = (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT) != 0;
vidMemBlock->mutex = gcvNULL;
vidMemBlock->physical = gcvNULL;
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
vidMemBlock->pageTables[i] = gcvNULL;
}
/* Allocate the mutex. */
gcmkONERROR(gckOS_CreateMutex(os, &vidMemBlock->mutex));
/* Allocate one gcuVIDMEM_NODE union. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer));
node = pointer;
if (!vidMemBlock->contiguous)
{
Flag |= gcvALLOC_FLAG_1M_PAGES;
}
/* Alloc 1M page size aligned memory block. */
gcmkONERROR(
gckOS_AllocatePagedMemory(os,
Flag,
&BlockSize,
&vidMemBlock->gid,
&vidMemBlock->physical));
/* Map current hardware mmu table with 1M pages for this video memory block. */
gcmkONERROR(gckVIDMEM_MapVidMemBlock(Kernel, vidMemBlock));
/* Initialize gcuVIDMEM_NODE union for virtual memory. */
node->VirtualChunk.kernel = Kernel;
node->VirtualChunk.offset = 0;
node->VirtualChunk.bytes = BlockSize;
node->VirtualChunk.kvaddr = gcvNULL;
node->VirtualChunk.logical = gcvNULL;
node->VirtualChunk.parent = vidMemBlock;
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
node->VirtualChunk.lockeds[i] = 0;
}
/* Initialize the virtual chunk linked-list. */
node->VirtualChunk.next =
node->VirtualChunk.prev =
node->VirtualChunk.nextFree =
node->VirtualChunk.prevFree = &vidMemBlock->node;
vidMemBlock->node.VirtualChunk.next =
vidMemBlock->node.VirtualChunk.prev =
vidMemBlock->node.VirtualChunk.nextFree =
vidMemBlock->node.VirtualChunk.prevFree = node;
vidMemBlock->node.VirtualChunk.bytes = 0;
*VidMemBlock = vidMemBlock;
gcmkFOOTER_ARG("*VidMemBlock=0x%x", *VidMemBlock);
return gcvSTATUS_OK;
OnError:
if (vidMemBlock != gcvNULL)
{
if (vidMemBlock->mutex)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(os, vidMemBlock->mutex));
}
if (vidMemBlock->physical)
{
gcmkVERIFY_OK(gckOS_FreePagedMemory(os,
vidMemBlock->physical,
vidMemBlock->bytes));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, vidMemBlock));
}
if (node != gcvNULL)
{
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node));
}
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_BLOCK_Destroy(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
gckDEVICE device = Kernel->device;
gctINT i;
gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p", Kernel, VidMemBlock);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(Kernel != gcvNULL);
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
if (VidMemBlock->physical)
{
gcmkVERIFY_OK(gckOS_FreePagedMemory(Kernel->os,
VidMemBlock->physical,
VidMemBlock->bytes));
}
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
if (VidMemBlock->pageTables[i])
{
gcmkVERIFY_OK(_UnmapVidMemBlock(device->mmus[i], i, VidMemBlock));
}
}
/* Free the mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, VidMemBlock->mutex));
/* Free the virtual chunk. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, VidMemBlock->node.VirtualChunk.next));
/* Free the video memory block. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, VidMemBlock));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
static gceSTATUS
_AllocateVirtualChunk(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock,
IN gceVIDMEM_TYPE Type,
INOUT gctSIZE_T *Bytes,
OUT gcuVIDMEM_NODE_PTR *Node
)
{
gceSTATUS status = gcvSTATUS_OK;
gctBOOL acquired = gcvFALSE;
gcuVIDMEM_NODE_PTR node;
gctSIZE_T bytes;
gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p Type=%x Bytes=%zx",
Kernel, VidMemBlock, Type, *Bytes);
gcmkVERIFY_ARGUMENT(Node != gcvNULL);
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Type < gcvVIDMEM_TYPE_COUNT);
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, VidMemBlock->mutex, gcvINFINITE));
acquired = gcvTRUE;
bytes = gcmALIGN(*Bytes, 4096);
if (bytes > VidMemBlock->freeBytes)
{
/* No enough memory. */
status = gcvSTATUS_OUT_OF_MEMORY;
goto OnError;
}
node = _FindVirtualChunkNode(Kernel, VidMemBlock, bytes);
if (node == gcvNULL)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
if (node->VirtualChunk.bytes > bytes)
{
/* Split the chunk. */
_SplitVirtualChunk(Kernel->os, node, bytes);
}
/* Remove the chunk from the free list. */
node->VirtualChunk.prevFree->VirtualChunk.nextFree = node->VirtualChunk.nextFree;
node->VirtualChunk.nextFree->VirtualChunk.prevFree = node->VirtualChunk.prevFree;
node->VirtualChunk.nextFree = node->VirtualChunk.prevFree = gcvNULL;
/* Fill in the information. */
node->VirtualChunk.parent = VidMemBlock;
VidMemBlock->freeBytes -= node->VirtualChunk.bytes;
*Bytes = bytes;
*Node = node;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, VidMemBlock->mutex));
}
return status;
}
static gceSTATUS
gckVIDMEM_AllocateVirtualChunk(
IN gckKERNEL Kernel,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
IN gctSIZE_T Bytes,
OUT gcuVIDMEM_NODE_PTR * Node
)
{
gckOS os;
gceSTATUS status;
gcuVIDMEM_NODE_PTR node;
gckVIDMEM_BLOCK vidMemBlock = gcvNULL;
gctSIZE_T blockSize;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%x Flag=%x Bytes=%lu", Kernel, Flag, Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Node != gcvNULL);
/* Extract the gckOS object pointer. */
os = Kernel->os;
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
/* Acquire the vidMem block mutex */
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->vidMemBlockMutex, gcvINFINITE));
acquired = gcvTRUE;
/* Find the free vidmem block. */
vidMemBlock = _FindFreeBlock(Kernel, Bytes);
if (!vidMemBlock)
{
/* Not found, construct new block. */
blockSize = gcmALIGN(Bytes, gcd1M_PAGE_SIZE);
gcmkONERROR(
gckVIDMEM_BLOCK_Construct(Kernel,
blockSize,
Type,
Flag,
&vidMemBlock));
gcmkONERROR(_AddToBlockList(Kernel, vidMemBlock));
}
/* Allocate virtual chunk node in the found block. */
gcmkONERROR(
_AllocateVirtualChunk(Kernel,
vidMemBlock,
Type,
&Bytes,
&node));
/* Return pointer to the gcuVIDMEM_NODE union. */
*Node = node;
/* Release the vidMem block mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex));
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Created virtual node 0x%x for %u bytes @ 0x%x",
node, Bytes, node->Virtual.physical);
/* Success. */
gcmkFOOTER_ARG("*Node=0x%x", *Node);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the vidMem block mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_Free
**
** Free an allocated video memory node.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to a gcuVIDMEM_NODE object.
**
** OUTPUT:
**
** Nothing.
*/
static gceSTATUS
gckVIDMEM_Free(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node
)
{
gceSTATUS status;
gckKERNEL kernel = gcvNULL;
gckVIDMEM memory = gcvNULL;
gckVIDMEM_BLOCK vidMemBlock = gcvNULL;
gcuVIDMEM_NODE_PTR node;
gctBOOL mutexAcquired = gcvFALSE;
gctBOOL vbMutexAcquired = gcvFALSE;
gctBOOL vbListMutexAcquired = gcvFALSE;
gctUINT64 mappingInOne = 1;
gcmkHEADER_ARG("Node=0x%x", Node);
/* Verify the arguments. */
if ((Node == gcvNULL)
|| (Node->VidMem.parent == gcvNULL)
)
{
/* Invalid object. */
gcmkONERROR(gcvSTATUS_INVALID_OBJECT);
}
vidMemBlock = Node->VirtualChunk.parent;
/**************************** Video Memory ********************************/
if (Node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
/* Extract pointer to gckVIDMEM object owning the node. */
memory = Node->VidMem.parent;
/* Acquire the mutex. */
gcmkONERROR(
gckOS_AcquireMutex(memory->os, memory->mutex, gcvINFINITE));
mutexAcquired = gcvTRUE;
if (Node->VidMem.kvaddr)
{
#if gcdCAPTURE_ONLY_MODE
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Node->VidMem.kvaddr));
#else
gcmkONERROR(
gckOS_DestroyKernelMapping(Kernel->os,
Node->VidMem.parent->physical,
Node->VidMem.kvaddr));
#endif
Node->VidMem.kvaddr = gcvNULL;
}
#ifdef __QNXNTO__
/* Unmap the video memory. */
if (Node->VidMem.logical != gcvNULL)
{
gckKERNEL_UnmapVideoMemory(
Kernel,
Node->VidMem.pool,
Node->VidMem.physical,
Node->VidMem.logical,
Node->VidMem.processID,
Node->VidMem.bytes
);
Node->VidMem.logical = gcvNULL;
}
/* Reset. */
Node->VidMem.processID = 0;
/* Don't try to re-free an already freed node. */
if ((Node->VidMem.nextFree == gcvNULL)
&& (Node->VidMem.prevFree == gcvNULL)
)
#endif
{
/* Check if Node is already freed. */
if (Node->VidMem.nextFree)
{
/* Node is alread freed. */
gcmkONERROR(gcvSTATUS_INVALID_DATA);
}
/* Update the number of free bytes. */
memory->freeBytes += Node->VidMem.bytes;
/* Find the next free node. */
for (node = Node->VidMem.next;
node != gcvNULL && node->VidMem.nextFree == gcvNULL;
node = node->VidMem.next) ;
if (node == gcvNULL)
{
gcmkONERROR(gcvSTATUS_INVALID_DATA);
}
/* Insert this node in the free list. */
Node->VidMem.nextFree = node;
Node->VidMem.prevFree = node->VidMem.prevFree;
Node->VidMem.prevFree->VidMem.nextFree =
node->VidMem.prevFree = Node;
/* Is the next node a free node and not the sentinel? */
if ((Node->VidMem.next == Node->VidMem.nextFree)
&& (Node->VidMem.next->VidMem.bytes != 0)
)
{
/* Merge this node with the next node. */
gcmkONERROR(_Merge(memory->os, node = Node));
gcmkASSERT(node->VidMem.nextFree != node);
gcmkASSERT(node->VidMem.prevFree != node);
}
/* Is the previous node a free node and not the sentinel? */
if ((Node->VidMem.prev == Node->VidMem.prevFree)
&& (Node->VidMem.prev->VidMem.bytes != 0)
)
{
/* Merge this node with the previous node. */
gcmkONERROR(_Merge(memory->os, node = Node->VidMem.prev));
gcmkASSERT(node->VidMem.nextFree != node);
gcmkASSERT(node->VidMem.prevFree != node);
}
}
gckOS_QueryOption(memory->os, "allMapInOne", &mappingInOne);
if (!mappingInOne)
{
gckOS_ReleaseReservedMemory(memory->os, Node->VidMem.physical);
Node->VidMem.physical = gcvNULL;
}
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(memory->os, memory->mutex));
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Node 0x%x is freed.",
Node);
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
/* Acquire the vidMem block mutex */
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->vidMemBlockMutex, gcvINFINITE));
vbListMutexAcquired = gcvTRUE;
if (vidMemBlock)
{
gckOS os = vidMemBlock->os;
gcmkONERROR(gckOS_AcquireMutex(os, vidMemBlock->mutex, gcvINFINITE));
vbMutexAcquired = gcvTRUE;
kernel = Node->VirtualChunk.kernel;
if (Node->VirtualChunk.kvaddr)
{
gcmkONERROR(
gckOS_DestroyKernelMapping(kernel->os,
vidMemBlock->physical,
Node->VirtualChunk.kvaddr));
Node->VirtualChunk.kvaddr = gcvNULL;
}
/* Handle the free chunk in the linked-list */
{
/* Check if chunk is in free list. */
if (Node->VirtualChunk.nextFree)
{
/* Chunk is already freed. */
gcmkONERROR(gcvSTATUS_INVALID_DATA);
}
vidMemBlock->freeBytes += Node->VirtualChunk.bytes;
/* Find the next free chunk. */
for (node = Node->VirtualChunk.next;
node != gcvNULL && node->VirtualChunk.nextFree == gcvNULL;
node = node->VirtualChunk.next);
if (node == gcvNULL)
{
gcmkONERROR(gcvSTATUS_INVALID_DATA);
}
/* Insert this chunk in the free list. */
Node->VirtualChunk.nextFree = node;
Node->VirtualChunk.prevFree = node->VirtualChunk.prevFree;
Node->VirtualChunk.prevFree->VirtualChunk.nextFree =
node->VirtualChunk.prevFree = Node;
/* Is the next chunk a free chunk. */
if ((Node->VirtualChunk.next == Node->VirtualChunk.nextFree)
&& (Node->VirtualChunk.next->VirtualChunk.bytes != 0)
)
{
/* Merge this chunk with the next chunk. */
gcmkONERROR(_MergeVirtualChunk(os, node = Node));
gcmkASSERT(node->VirtualChunk.nextFree != node);
gcmkASSERT(node->VirtualChunk.prevFree != node);
}
/* Is the previous chunk a free chunk. */
if ((Node->VirtualChunk.prev == Node->VirtualChunk.prevFree)
&& (Node->VirtualChunk.prev->VirtualChunk.bytes != 0)
)
{
/* Merge this chunk with the previous chunk. */
gcmkONERROR(_MergeVirtualChunk(os, node = Node->VirtualChunk.prev));
gcmkASSERT(node->VirtualChunk.nextFree != node);
gcmkASSERT(node->VirtualChunk.prevFree != node);
}
}
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, vidMemBlock->mutex));
/* Only free the vidmem block when all the chunks are freed. */
if (_IsVidMemBlockFree(vidMemBlock))
{
gcmkONERROR(_RemoveFromBlockList(kernel, vidMemBlock));
gcmkONERROR(gckVIDMEM_BLOCK_Destroy(kernel, vidMemBlock));
}
}
/* Release the vidMem block mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*************************** Virtual Memory *******************************/
/* Get gckKERNEL object. */
kernel = Node->Virtual.kernel;
/* Verify the gckKERNEL object pointer. */
gcmkVERIFY_OBJECT(kernel, gcvOBJ_KERNEL);
if (Node->Virtual.kvaddr)
{
gcmkVERIFY_OK(
gckOS_DestroyKernelMapping(kernel->os,
Node->Virtual.physical,
Node->Virtual.kvaddr));
}
/* Free the virtual memory. */
gcmkVERIFY_OK(gckOS_FreePagedMemory(kernel->os,
Node->Virtual.physical,
Node->Virtual.bytes));
/* Delete the gcuVIDMEM_NODE union. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(kernel->os, Node));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (mutexAcquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(
memory->os, memory->mutex
));
}
if (vbMutexAcquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(
vidMemBlock->os, vidMemBlock->mutex
));
}
if (vbListMutexAcquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(
vidMemBlock->os, Kernel->vidMemBlockMutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_Lock
**
** Lock a video memory node and return its hardware specific address.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to a gcuVIDMEM_NODE union.
**
** OUTPUT:
**
** gctUINT32 * Address
** Pointer to a variable that will hold the hardware specific address.
**
** gctUINT32 * PhysicalAddress
** Pointer to a variable that will hold the bus address of a contiguous
** video node.
*/
static gceSTATUS
gckVIDMEM_Lock(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
OUT gctUINT32 * Address
)
{
gckOS os;
gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node);
/* Extract the gckOS object pointer. */
os = Kernel->os;
/* Increment the lock count. */
if (Node->VidMem.locked++ == 0)
{
gctUINT32 address;
gctUINT32 offset = (gctUINT32)Node->VidMem.offset;
switch (Node->VidMem.pool)
{
case gcvPOOL_LOCAL_EXTERNAL:
address = Kernel->externalBaseAddress + offset;
break;
case gcvPOOL_LOCAL_INTERNAL:
address = Kernel->internalBaseAddress + offset;
break;
case gcvPOOL_INTERNAL_SRAM:
address = Kernel->sRAMBaseAddresses[Kernel->sRAMIndex] + offset;
break;
case gcvPOOL_EXTERNAL_SRAM:
address = Kernel->extSRAMBaseAddresses[Kernel->extSRAMIndex] + offset;
break;
default:
gcmkASSERT(Node->VidMem.pool == gcvPOOL_SYSTEM);
/*FALLTHRU*/
case gcvPOOL_SYSTEM:
address = Kernel->contiguousBaseAddress + offset;
break;
}
/* Save address. */
Node->VidMem.address = address;
}
*Address = Node->VidMem.address;
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Locked node 0x%x (%d) @ 0x%08X",
Node,
Node->VidMem.locked,
*Address);
gcmkFOOTER_ARG("*Address=0x%08X", *Address);
return gcvSTATUS_OK;
}
static gceSTATUS
gckVIDMEM_LockVirtual(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
OUT gctUINT32 * Address
)
{
gceSTATUS status;
gctPHYS_ADDR_T physicalAddress;
gctBOOL locked = gcvFALSE;
gckOS os = Kernel->os;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node);
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
gcmkONERROR(
gckOS_GetPhysicalFromHandle(os,
Node->Virtual.physical,
0,
&physicalAddress));
/* Increment the lock count. */
if (Node->Virtual.lockeds[hwType]++ == 0)
{
locked = gcvTRUE;
status = _ConvertPhysical(
Kernel,
Kernel->core,
Node,
gcvNULL,
physicalAddress,
&Node->Virtual.addresses[hwType]
);
if (gcmIS_ERROR(status))
{
/* Do GPU address mapping. */
#if gcdSECURITY
gctPHYS_ADDR physicalArrayPhysical;
gctPOINTER physicalArrayLogical;
gcmkONERROR(gckOS_AllocatePageArray(
os,
Node->Virtual.physical,
Node->Virtual.pageCount,
&physicalArrayLogical,
&physicalArrayPhysical
));
gcmkONERROR(gckKERNEL_SecurityMapMemory(
Kernel,
physicalArrayLogical,
Node->Virtual.pageCount,
&Node->Virtual.addresses[hwType]
));
gcmkONERROR(gckOS_FreeNonPagedMemory(
os,
physicalArrayPhysical,
physicalArrayLogical,
1
));
#else
{
/* Allocate pages inside the MMU. */
gcmkONERROR(
gckMMU_AllocatePagesEx(Kernel->mmu,
Node->Virtual.pageCount,
Node->Virtual.type,
gcvPAGE_TYPE_4K,
Node->Virtual.secure,
&Node->Virtual.pageTables[hwType],
&Node->Virtual.addresses[hwType]));
}
if (Node->Virtual.onFault != gcvTRUE)
{
#if gcdENABLE_TRUST_APPLICATION
if (Kernel->hardware->options.secureMode == gcvSECURE_IN_TA)
{
gcmkONERROR(gckKERNEL_MapInTrustApplicaiton(
Kernel,
Node->Virtual.logical,
Node->Virtual.physical,
Node->Virtual.addresses[hwType],
Node->Virtual.pageCount
));
}
else
#endif
{
gcmkDUMP(os, "#[mmu: dynamic mapping: address=0x%08X pageCount=%lu]",
Node->Virtual.addresses[hwType],
(unsigned long)Node->Virtual.pageCount);
/* Map the pages. */
gcmkONERROR(gckOS_MapPagesEx(os,
Kernel->core,
Node->Virtual.physical,
Node->Virtual.pageCount,
Node->Virtual.addresses[hwType],
Node->Virtual.pageTables[hwType],
gcvTRUE,
Node->Virtual.type));
}
}
{
gcmkONERROR(gckMMU_Flush(Kernel->mmu, Node->Virtual.type));
}
#endif
/* GPU MMU page size is fixed at 4096 now. */
Node->Virtual.addresses[hwType] |= (gctUINT32)physicalAddress & (4096 - 1);
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Mapped virtual node 0x%x to 0x%08X",
Node,
Node->Virtual.addresses[hwType]);
}
/* Return hardware address. */
*Address = Node->Virtual.addresses[hwType];
gcmkFOOTER_ARG("*Address=0x%08X", *Address);
return gcvSTATUS_OK;
OnError:
if (locked)
{
if (Node->Virtual.pageTables[hwType] != gcvNULL)
{
{
/* Free the pages from the MMU. */
gcmkVERIFY_OK(
gckMMU_FreePages(Kernel->mmu,
Node->Virtual.secure,
gcvPAGE_TYPE_4K,
Node->Virtual.addresses[hwType],
Node->Virtual.pageTables[hwType],
Node->Virtual.pageCount));
}
Node->Virtual.pageTables[hwType] = gcvNULL;
}
Node->Virtual.lockeds[hwType]--;
}
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_LockVirtualChunk(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
OUT gctUINT32 * Address
)
{
gceSTATUS status = gcvSTATUS_OK;
gckVIDMEM_BLOCK vidMemBlock = Node->VirtualChunk.parent;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node);
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
gcmkASSERT(vidMemBlock != gcvNULL);
/* Increment the lock count. */
if (Node->VirtualChunk.lockeds[hwType]++ == 0)
{
if (!vidMemBlock->pageTables[hwType])
{
/* Map current hardware mmu table with 1M pages for this video memory block. */
gcmkONERROR(gckVIDMEM_MapVidMemBlock(Kernel, vidMemBlock));
}
Node->VirtualChunk.addresses[hwType] = vidMemBlock->addresses[hwType]
+ (gctUINT32)Node->VirtualChunk.offset;
}
/* Return hardware address. */
*Address = Node->VirtualChunk.addresses[hwType];
gcmkFOOTER_ARG("*Address=0x%08X", *Address);
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_Unlock
**
** Unlock a video memory node.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to a locked gcuVIDMEM_NODE union.
**
** gctBOOL * Asynchroneous
** Pointer to a variable specifying whether the surface should be
** unlocked asynchroneously or not.
**
** OUTPUT:
**
** gctBOOL * Asynchroneous
** Pointer to a variable receiving the number of bytes used in the
** command buffer specified by 'Commands'. If gcvNULL, there is no
** command buffer.
*/
static gceSTATUS
gckVIDMEM_Unlock(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
IN OUT gctBOOL * Asynchroneous
)
{
gceSTATUS status;
gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d",
Node, gcmOPT_VALUE(Asynchroneous));
if (Node->VidMem.locked <= 0)
{
/* The surface was not locked. */
gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED);
}
if (Asynchroneous != gcvNULL)
{
/* Schedule an event to sync with GPU. */
*Asynchroneous = gcvTRUE;
}
else
{
/* Decrement the lock count. */
Node->VidMem.locked--;
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Unlocked node %p (%d)",
Node,
Node->VidMem.locked);
/* Success. */
gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous));
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_UnlockVirtual(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
IN OUT gctBOOL * Asynchroneous
)
{
gceSTATUS status;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d",
Node, gcmOPT_VALUE(Asynchroneous));
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
if (Asynchroneous != gcvNULL)
{
/* Schedule the surface to be unlocked. */
*Asynchroneous = gcvTRUE;
}
else
{
if (Node->Virtual.lockeds[hwType] == 0)
{
gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED);
}
/* Decrement lock count. */
--Node->Virtual.lockeds[hwType];
/* See if we can unlock the resources. */
if (Node->Virtual.lockeds[hwType] == 0)
{
gctUINT32 address;
/* Adjust address to page aligned for underlying functions. */
address = Node->Virtual.addresses[hwType] & ~(4096 - 1);
#if gcdSECURITY
if (Node->Virtual.addresses[hwType] > 0x80000000)
{
gcmkONERROR(gckKERNEL_SecurityUnmapMemory(
Kernel,
address,
Node->Virtual.pageCount
));
}
#else
/* Free the page table. */
if (Node->Virtual.pageTables[hwType] != gcvNULL)
{
{
gcmkONERROR(
gckMMU_FreePages(Kernel->mmu,
Node->Virtual.secure,
gcvPAGE_TYPE_4K,
address,
Node->Virtual.pageTables[hwType],
Node->Virtual.pageCount));
}
gcmkONERROR(gckOS_UnmapPages(
Kernel->os,
Node->Virtual.pageCount,
address
));
/* Mark page table as freed. */
Node->Virtual.pageTables[hwType] = gcvNULL;
}
#endif
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Unmapped virtual node %p from 0x%08X",
Node, Node->Virtual.addresses[hwType]);
}
/* Success. */
gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous));
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_UnlockVirtualChunk(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
IN OUT gctBOOL * Asynchroneous
)
{
gceSTATUS status;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d",
Node, gcmOPT_VALUE(Asynchroneous));
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
if (Asynchroneous != gcvNULL)
{
/* Schedule an event to sync with GPU. */
*Asynchroneous = gcvTRUE;
}
else
{
if (Node->VirtualChunk.lockeds[hwType] == 0)
{
/* The surface was not locked. */
gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED);
}
/* Unmap and free pages when video memory free. */
/* Decrement the lock count. */
--Node->VirtualChunk.lockeds[hwType];
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Unlocked node %p (%d)",
Node,
Node->VirtualChunk.lockeds[hwType]);
/* Success. */
gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous));
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_HANDLE_Allocate
**
** Allocate a handle for a gckVIDMEM_NODE object.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gckVIDMEM_NODE Node
** Pointer to a gckVIDMEM_NODE object.
**
** OUTPUT:
**
** gctUINT32 * Handle
** Pointer to a variable receiving a handle represent this
** gckVIDMEM_NODE in userspace.
*/
gceSTATUS
gckVIDMEM_HANDLE_Allocate(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE Node,
OUT gctUINT32 * Handle
)
{
gceSTATUS status;
gctUINT32 processID = 0;
gctPOINTER pointer = gcvNULL;
gctPOINTER handleDatabase = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctUINT32 handle = 0;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gckOS os = Kernel->os;
gcmkHEADER_ARG("Kernel=0x%X, Node=0x%X", Kernel, Node);
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
/* Allocate a gckVIDMEM_HANDLE object. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_HANDLE), &pointer));
gcmkVERIFY_OK(gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsVIDMEM_HANDLE)));
handleObject = pointer;
gcmkONERROR(gckOS_AtomConstruct(os, &handleObject->reference));
/* Set default reference count to 1. */
gckOS_AtomSet(os, handleObject->reference, 1);
gcmkVERIFY_OK(gckOS_GetProcessID(&processID));
gcmkONERROR(
gckKERNEL_FindHandleDatbase(Kernel,
processID,
&handleDatabase,
&mutex));
/* Allocate a handle for this object. */
gcmkONERROR(
gckKERNEL_AllocateIntegerId(handleDatabase, handleObject, &handle));
handleObject->node = Node;
handleObject->handle = handle;
*Handle = handle;
gcmkFOOTER_ARG("*Handle=%d", *Handle);
return gcvSTATUS_OK;
OnError:
if (handleObject != gcvNULL)
{
if (handleObject->reference != gcvNULL)
{
gcmkVERIFY_OK(gckOS_AtomDestroy(os, handleObject->reference));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, handleObject));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_HANDLE_Reference(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Handle
)
{
gceSTATUS status;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gctPOINTER database = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctINT32 oldValue = 0;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Handle=%d PrcoessID=%d", Handle, ProcessID);
gcmkONERROR(
gckKERNEL_FindHandleDatbase(Kernel, ProcessID, &database, &mutex));
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
/* Translate handle to gckVIDMEM_HANDLE object. */
gcmkONERROR(
gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject));
/* Increase the reference count. */
gckOS_AtomIncrement(Kernel->os, handleObject->reference, &oldValue);
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_HANDLE_Dereference(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Handle
)
{
gceSTATUS status;
gctPOINTER handleDatabase = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctINT32 oldValue = 0;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Handle=%d PrcoessID=%d", Handle, ProcessID);
gcmkONERROR(
gckKERNEL_FindHandleDatbase(Kernel,
ProcessID,
&handleDatabase,
&mutex));
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
/* Translate handle to gckVIDMEM_HANDLE. */
gcmkONERROR(
gckKERNEL_QueryIntegerId(handleDatabase, Handle, (gctPOINTER *)&handleObject));
gckOS_AtomDecrement(Kernel->os, handleObject->reference, &oldValue);
if (oldValue == 1)
{
/* Remove handle from database if this is the last reference. */
gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(handleDatabase, Handle));
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
if (oldValue == 1)
{
gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, handleObject->reference));
gcmkOS_SAFE_FREE(Kernel->os, handleObject);
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_HANDLE_Lookup(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Handle,
OUT gckVIDMEM_NODE * Node
)
{
gceSTATUS status;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gckVIDMEM_NODE node = gcvNULL;
gctPOINTER database = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X ProcessID=%d Handle=%d",
Kernel, ProcessID, Handle);
gcmkONERROR(
gckKERNEL_FindHandleDatbase(Kernel, ProcessID, &database, &mutex));
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
gcmkONERROR(
gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject));
node = handleObject->node;
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
*Node = node;
gcmkFOOTER_ARG("*Node=%d", *Node);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_HANDLE_Lookup2(
IN gckKERNEL Kernel,
IN gcsDATABASE_PTR Database,
IN gctUINT32 Handle,
OUT gckVIDMEM_NODE * Node
)
{
gceSTATUS status;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gckVIDMEM_NODE node = gcvNULL;
gctPOINTER database = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X Database=%p Handle=%d",
Kernel, Database, Handle);
database = Database->handleDatabase;
mutex = Database->handleDatabaseMutex;
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
gcmkONERROR(
gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject));
node = handleObject->node;
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
*Node = node;
gcmkFOOTER_ARG("*Node=%d", *Node);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_NODE_Construct(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR VideoNode,
IN gceVIDMEM_TYPE Type,
IN gcePOOL Pool,
OUT gckVIDMEM_NODE * NodeObject
)
{
gceSTATUS status;
gckVIDMEM_NODE node = gcvNULL;
gctPOINTER pointer = gcvNULL;
gckOS os = Kernel->os;
gctUINT i;
/* Construct a node. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_NODE), &pointer));
gcmkVERIFY_OK(gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsVIDMEM_NODE)));
node = pointer;
node->metadata.magic = VIV_VIDMEM_METADATA_MAGIC;
node->metadata.ts_fd = -1;
#ifdef gcdANDROID
node->metadata.ts_address = 0;
#endif
node->node = VideoNode;
node->kernel = Kernel;
node->type = Type;
node->pool = Pool;
gcmkONERROR(gckOS_AtomConstruct(os, &node->reference));
gcmkONERROR(gckOS_CreateMutex(os, &node->mutex));
for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
{
gcmkONERROR(gckOS_CreateSignal(os, gcvFALSE, &node->sync[i].signal));
}
/* Reference is 1 by default . */
gckOS_AtomSet(os, node->reference, 1);
gcmkVERIFY_OK(
gckOS_AcquireMutex(Kernel->os,
Kernel->db->videoMemListMutex,
gcvINFINITE));
/* Add into video memory node list. */
gcsLIST_Add(&node->link, &Kernel->db->videoMemList);
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex));
*NodeObject = node;
return gcvSTATUS_OK;
OnError:
if (node != gcvNULL)
{
if (node->mutex)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(os, node->mutex));
}
if (node->reference != gcvNULL)
{
gcmkVERIFY_OK(gckOS_AtomDestroy(os, node->reference));
}
for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
{
if (node->sync[i].signal != gcvNULL)
{
gcmkVERIFY_OK(gckOS_DestroySignal(os, node->sync[i].signal));
}
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node));
}
return status;
}
gceSTATUS
gckVIDMEM_NODE_AllocateLinear(
IN gckKERNEL Kernel,
IN gckVIDMEM VideoMemory,
IN gcePOOL Pool,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
IN gctUINT32 Alignment,
IN gctBOOL Specified,
IN OUT gctSIZE_T * Bytes,
OUT gckVIDMEM_NODE * NodeObject
)