Google Git
Sign in
coral / imx-gpu-viv-ko / refs/heads/master / . / hal / kernel / gc_hal_kernel_video_memory.c
blob: 7e3b93cfc3f870d6c889e4efcda88776566ab463 [file] [log] [blame] [edit]
/****************************************************************************
*
* The MIT License (MIT)
*
* Copyright (c) 2014 - 2020 Vivante Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
*****************************************************************************
*
* The GPL License (GPL)
*
* Copyright (C) 2014 - 2020 Vivante Corporation
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*****************************************************************************
*
* Note: This software is released under dual MIT and GPL licenses. A
* recipient may use this file under the terms of either the MIT license or
* GPL License. If you wish to use only one license not the other, you can
* indicate your decision by deleting one of the above license notices in your
* version of this file.
*
*****************************************************************************/
#include "gc_hal_kernel_precomp.h"
#define _GC_OBJ_ZONE gcvZONE_VIDMEM
/******************************************************************************\
******************************* Private Functions ******************************
\******************************************************************************/
/*******************************************************************************
**
** _Split
**
** Split a node on the required byte boundary.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to the node to split.
**
** gctSIZE_T Bytes
** Number of bytes to keep in the node.
**
** OUTPUT:
**
** Nothing.
**
** RETURNS:
**
** gctBOOL
** gcvTRUE if the node was split successfully, or gcvFALSE if there is an
** error.
**
*/
static gctBOOL
_Split(
IN gckOS Os,
IN gcuVIDMEM_NODE_PTR Node,
IN gctSIZE_T Bytes
)
{
gcuVIDMEM_NODE_PTR node;
gctPOINTER pointer = gcvNULL;
/* Make sure the byte boundary makes sense. */
if ((Bytes <= 0) || (Bytes > Node->VidMem.bytes))
{
return gcvFALSE;
}
/* Allocate a new gcuVIDMEM_NODE object. */
if (gcmIS_ERROR(gckOS_Allocate(Os,
gcmSIZEOF(gcuVIDMEM_NODE),
&pointer)))
{
/* Error. */
return gcvFALSE;
}
node = pointer;
/* Initialize gcuVIDMEM_NODE structure. */
node->VidMem.offset = Node->VidMem.offset + Bytes;
node->VidMem.bytes = Node->VidMem.bytes - Bytes;
node->VidMem.alignment = 0;
node->VidMem.locked = 0;
node->VidMem.parent = Node->VidMem.parent;
node->VidMem.pool = Node->VidMem.pool;
node->VidMem.processID = 0;
node->VidMem.logical = gcvNULL;
node->VidMem.kvaddr = gcvNULL;
/* Insert node behind specified node. */
node->VidMem.next = Node->VidMem.next;
node->VidMem.prev = Node;
Node->VidMem.next = node->VidMem.next->VidMem.prev = node;
/* Insert free node behind specified node. */
node->VidMem.nextFree = Node->VidMem.nextFree;
node->VidMem.prevFree = Node;
Node->VidMem.nextFree = node->VidMem.nextFree->VidMem.prevFree = node;
/* Adjust size of specified node. */
Node->VidMem.bytes = Bytes;
/* Success. */
return gcvTRUE;
}
/*******************************************************************************
**
** _Merge
**
** Merge two adjacent nodes together.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to the first of the two nodes to merge.
**
** OUTPUT:
**
** Nothing.
**
*/
static gceSTATUS
_Merge(
IN gckOS Os,
IN gcuVIDMEM_NODE_PTR Node
)
{
gcuVIDMEM_NODE_PTR node;
gceSTATUS status;
/* Save pointer to next node. */
node = Node->VidMem.next;
/* This is a good time to make sure the heap is not corrupted. */
if (Node->VidMem.offset + Node->VidMem.bytes != node->VidMem.offset)
{
/* Corrupted heap. */
gcmkASSERT(
Node->VidMem.offset + Node->VidMem.bytes == node->VidMem.offset);
return gcvSTATUS_HEAP_CORRUPTED;
}
/* Adjust byte count. */
Node->VidMem.bytes += node->VidMem.bytes;
/* Unlink next node from linked list. */
Node->VidMem.next = node->VidMem.next;
Node->VidMem.nextFree = node->VidMem.nextFree;
Node->VidMem.next->VidMem.prev =
Node->VidMem.nextFree->VidMem.prevFree = Node;
/* Free next node. */
status = gcmkOS_SAFE_FREE(Os, node);
return status;
}
/******************************************************************************\
******************************* gckVIDMEM API Code ******************************
\******************************************************************************/
/*******************************************************************************
**
** gckVIDMEM_Construct
**
** Construct a new gckVIDMEM object.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR_T PhysicalBase
** Base physical address for the video memory heap.
**
** gctSIZE_T Bytes
** Number of bytes in the video memory heap.
**
** gctSIZE_T Threshold
** Minimum number of bytes beyond am allocation before the node is
** split. Can be used as a minimum alignment requirement.
**
** gctSIZE_T BankSize
** Number of bytes per physical memory bank. Used by bank
** optimization.
**
** OUTPUT:
**
** gckVIDMEM * Memory
** Pointer to a variable that will hold the pointer to the gckVIDMEM
** object.
*/
gceSTATUS
gckVIDMEM_Construct(
IN gckOS Os,
IN gctPHYS_ADDR_T PhysicalBase,
IN gctSIZE_T Bytes,
IN gctSIZE_T Threshold,
IN gctSIZE_T BankSize,
OUT gckVIDMEM * Memory
)
{
gckVIDMEM memory = gcvNULL;
gceSTATUS status;
gcuVIDMEM_NODE_PTR node;
gctINT i, banks = 0;
gctPOINTER pointer = gcvNULL;
gctUINT32 heapBytes;
gctUINT32 bankSize;
gctUINT32 base = 0;
gcmkHEADER_ARG("Os=0x%x PhysicalBase=%12llx Bytes=%lu Threshold=%lu "
"BankSize=%lu",
Os, PhysicalBase, Bytes, Threshold, BankSize);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Memory != gcvNULL);
gcmkSAFECASTSIZET(heapBytes, Bytes);
gcmkSAFECASTSIZET(bankSize, BankSize);
/* Allocate the gckVIDMEM object. */
gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(struct _gckVIDMEM), &pointer));
gckOS_ZeroMemory(pointer, gcmSIZEOF(struct _gckVIDMEM));
memory = pointer;
/* Initialize the gckVIDMEM object. */
memory->object.type = gcvOBJ_VIDMEM;
memory->os = Os;
/* Set video memory heap information. */
memory->physicalBase = PhysicalBase;
memory->bytes = heapBytes;
memory->freeBytes = heapBytes;
memory->minFreeBytes = heapBytes;
memory->capability = ~0u;
memory->threshold = Threshold;
memory->mutex = gcvNULL;
/* Walk all possible banks. */
for (i = 0; i < gcmCOUNTOF(memory->sentinel); ++i)
{
gctUINT32 bytes;
if (BankSize == 0)
{
/* Use all bytes for the first bank. */
bytes = heapBytes;
}
else
{
/* Compute number of bytes for this bank. */
bytes = gcmALIGN(base + 1, bankSize) - base;
if (bytes > heapBytes)
{
/* Make sure we don't exceed the total number of bytes. */
bytes = heapBytes;
}
}
if (bytes == 0)
{
/* Mark heap is not used. */
memory->sentinel[i].VidMem.next =
memory->sentinel[i].VidMem.prev =
memory->sentinel[i].VidMem.nextFree =
memory->sentinel[i].VidMem.prevFree = gcvNULL;
continue;
}
/* Allocate one gcuVIDMEM_NODE union. */
gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer));
node = pointer;
/* Initialize gcuVIDMEM_NODE union. */
node->VidMem.parent = memory;
node->VidMem.next =
node->VidMem.prev =
node->VidMem.nextFree =
node->VidMem.prevFree = &memory->sentinel[i];
node->VidMem.offset = base;
node->VidMem.bytes = bytes;
node->VidMem.alignment = 0;
node->VidMem.pool = gcvPOOL_UNKNOWN;
node->VidMem.locked = 0;
node->VidMem.processID = 0;
node->VidMem.logical = gcvNULL;
node->VidMem.kvaddr = gcvNULL;
/* Initialize the linked list of nodes. */
memory->sentinel[i].VidMem.next =
memory->sentinel[i].VidMem.prev =
memory->sentinel[i].VidMem.nextFree =
memory->sentinel[i].VidMem.prevFree = node;
/* Mark sentinel. */
memory->sentinel[i].VidMem.bytes = 0;
/* Adjust address for next bank. */
base += bytes;
heapBytes -= bytes;
banks ++;
}
/* Assign all the bank mappings. */
memory->mapping[gcvVIDMEM_TYPE_COLOR_BUFFER] = banks - 1;
memory->mapping[gcvVIDMEM_TYPE_BITMAP] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_DEPTH_BUFFER] = banks - 1;
memory->mapping[gcvVIDMEM_TYPE_HZ_BUFFER] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_TEXTURE] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_VERTEX_BUFFER] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_INDEX_BUFFER] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_TILE_STATUS] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_COMMAND] = banks - 1;
if (banks > 1) --banks;
memory->mapping[gcvVIDMEM_TYPE_GENERIC] = 0;
memory->mapping[gcvVIDMEM_TYPE_ICACHE] = 0;
memory->mapping[gcvVIDMEM_TYPE_TXDESC] = 0;
memory->mapping[gcvVIDMEM_TYPE_FENCE] = 0;
memory->mapping[gcvVIDMEM_TYPE_TFBHEADER] = 0;
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] INDEX: bank %d",
memory->mapping[gcvVIDMEM_TYPE_INDEX_BUFFER]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] VERTEX: bank %d",
memory->mapping[gcvVIDMEM_TYPE_VERTEX_BUFFER]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] TEXTURE: bank %d",
memory->mapping[gcvVIDMEM_TYPE_TEXTURE]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] RENDER_TARGET: bank %d",
memory->mapping[gcvVIDMEM_TYPE_COLOR_BUFFER]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] DEPTH: bank %d",
memory->mapping[gcvVIDMEM_TYPE_DEPTH_BUFFER]);
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"[GALCORE] TILE_STATUS: bank %d",
memory->mapping[gcvVIDMEM_TYPE_TILE_STATUS]);
/* Allocate the mutex. */
gcmkONERROR(gckOS_CreateMutex(Os, &memory->mutex));
/* Return pointer to the gckVIDMEM object. */
*Memory = memory;
/* Success. */
gcmkFOOTER_ARG("*Memory=0x%x", *Memory);
return gcvSTATUS_OK;
OnError:
/* Roll back. */
if (memory != gcvNULL)
{
if (memory->mutex != gcvNULL)
{
/* Delete the mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Os, memory->mutex));
}
for (i = 0; i < banks; ++i)
{
/* Free the heap. */
gcmkASSERT(memory->sentinel[i].VidMem.next != gcvNULL);
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory->sentinel[i].VidMem.next));
}
/* Free the object. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_Destroy
**
** Destroy an gckVIDMEM object.
**
** INPUT:
**
** gckVIDMEM Memory
** Pointer to an gckVIDMEM object to destroy.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckVIDMEM_Destroy(
IN gckVIDMEM Memory
)
{
gcuVIDMEM_NODE_PTR node, next;
gctINT i;
gcmkHEADER_ARG("Memory=0x%x", Memory);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM);
/* Walk all sentinels. */
for (i = 0; i < gcmCOUNTOF(Memory->sentinel); ++i)
{
/* Bail out of the heap is not used. */
if (Memory->sentinel[i].VidMem.next == gcvNULL)
{
break;
}
/* Walk all the nodes until we reach the sentinel. */
for (node = Memory->sentinel[i].VidMem.next;
node->VidMem.bytes != 0;
node = next)
{
/* Save pointer to the next node. */
next = node->VidMem.next;
/* Free the node. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, node));
}
}
/* Free the mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Memory->os, Memory->mutex));
/* Mark the object as unknown. */
Memory->object.type = gcvOBJ_UNKNOWN;
/* Free the gckVIDMEM object. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, Memory));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
#if gcdENABLE_BANK_ALIGNMENT
#if !gcdBANK_BIT_START
#error gcdBANK_BIT_START not defined.
#endif
#if !gcdBANK_BIT_END
#error gcdBANK_BIT_END not defined.
#endif
/*******************************************************************************
** _GetSurfaceBankAlignment
**
** Return the required offset alignment required to the make BaseAddress
** aligned properly.
**
** INPUT:
**
** gckOS Os
** Pointer to gcoOS object.
**
** gceVIDMEM_TYPE Type
** Type of allocation.
**
** gctUINT32 BaseAddress
** Base address of current video memory node.
**
** OUTPUT:
**
** gctUINT32_PTR AlignmentOffset
** Pointer to a variable that will hold the number of bytes to skip in
** the current video memory node in order to make the alignment bank
** aligned.
*/
static gceSTATUS
_GetSurfaceBankAlignment(
IN gckKERNEL Kernel,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 BaseAddress,
OUT gctUINT32_PTR AlignmentOffset
)
{
gctUINT32 bank;
/* To retrieve the bank. */
static const gctUINT32 bankMask = (0xFFFFFFFF << gcdBANK_BIT_START)
^ (0xFFFFFFFF << (gcdBANK_BIT_END + 1));
/* To retrieve the bank and all the lower bytes. */
static const gctUINT32 byteMask = ~(0xFFFFFFFF << (gcdBANK_BIT_END + 1));
gcmkHEADER_ARG("Type=%d BaseAddress=0x%x ", Type, BaseAddress);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(AlignmentOffset != gcvNULL);
switch (Type)
{
case gcvVIDMEM_TYPE_COLOR_BUFFER:
bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START);
/* Align to the first bank. */
*AlignmentOffset = (bank == 0) ?
0 :
((1 << (gcdBANK_BIT_END + 1)) + 0) - (BaseAddress & byteMask);
break;
case gcvVIDMEM_TYPE_DEPTH_BUFFER:
bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START);
/* Align to the third bank. */
*AlignmentOffset = (bank == 2) ?
0 :
((1 << (gcdBANK_BIT_END + 1)) + (2 << gcdBANK_BIT_START)) - (BaseAddress & byteMask);
/* Minimum 256 byte alignment needed for fast_msaa. */
if ((gcdBANK_CHANNEL_BIT > 7) ||
((gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_FAST_MSAA) != gcvSTATUS_TRUE) &&
(gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_SMALL_MSAA) != gcvSTATUS_TRUE)))
{
/* Add a channel offset at the channel bit. */
*AlignmentOffset += (1 << gcdBANK_CHANNEL_BIT);
}
break;
default:
/* no alignment needed. */
*AlignmentOffset = 0;
}
/* Return the status. */
gcmkFOOTER_ARG("*AlignmentOffset=%u", *AlignmentOffset);
return gcvSTATUS_OK;
}
#endif
static gcuVIDMEM_NODE_PTR
_FindNode(
IN gckKERNEL Kernel,
IN gckVIDMEM Memory,
IN gctINT Bank,
IN gctSIZE_T Bytes,
IN gceVIDMEM_TYPE Type,
IN OUT gctUINT32_PTR Alignment
)
{
gcuVIDMEM_NODE_PTR node;
gctUINT32 alignment;
#if gcdENABLE_BANK_ALIGNMENT
gctUINT32 bankAlignment;
gceSTATUS status;
#endif
if (Memory->sentinel[Bank].VidMem.nextFree == gcvNULL)
{
/* No free nodes left. */
return gcvNULL;
}
#if gcdENABLE_BANK_ALIGNMENT
/* Walk all free nodes until we have one that is big enough or we have
** reached the sentinel. */
for (node = Memory->sentinel[Bank].VidMem.nextFree;
node->VidMem.bytes != 0;
node = node->VidMem.nextFree)
{
if (node->VidMem.bytes < Bytes)
{
continue;
}
gcmkONERROR(_GetSurfaceBankAlignment(
Kernel,
Type,
(gctUINT32)(node->VidMem.parent->physicalBase + node->VidMem.offset),
&bankAlignment));
bankAlignment = gcmALIGN(bankAlignment, *Alignment);
/* Compute number of bytes to skip for alignment. */
alignment = (*Alignment == 0)
? 0
: (*Alignment - (node->VidMem.offset % *Alignment));
if (alignment == *Alignment)
{
/* Node is already aligned. */
alignment = 0;
}
if (node->VidMem.bytes >= Bytes + alignment + bankAlignment)
{
/* This node is big enough. */
*Alignment = alignment + bankAlignment;
return node;
}
}
#endif
/* Walk all free nodes until we have one that is big enough or we have
reached the sentinel. */
for (node = Memory->sentinel[Bank].VidMem.nextFree;
node->VidMem.bytes != 0;
node = node->VidMem.nextFree)
{
gctUINT offset;
gctINT modulo;
gcmkSAFECASTSIZET(offset, node->VidMem.offset);
modulo = gckMATH_ModuloInt(offset, *Alignment);
/* Compute number of bytes to skip for alignment. */
alignment = (*Alignment == 0) ? 0 : (*Alignment - modulo);
if (alignment == *Alignment)
{
/* Node is already aligned. */
alignment = 0;
}
if (node->VidMem.bytes >= Bytes + alignment)
{
/* This node is big enough. */
*Alignment = alignment;
return node;
}
}
#if gcdENABLE_BANK_ALIGNMENT
OnError:
#endif
/* Not enough memory. */
return gcvNULL;
}
/*******************************************************************************
**
** gckVIDMEM_AllocateLinear
**
** Allocate linear memory from the gckVIDMEM object.
**
** INPUT:
**
** gckVIDMEM Memory
** Pointer to an gckVIDMEM object.
**
** gctSIZE_T Bytes
** Number of bytes to allocate.
**
** gctUINT32 Alignment
** Byte alignment for allocation.
**
** gceVIDMEM_TYPE Type
** Type of surface to allocate (use by bank optimization).
**
** gctUINT32 Flag
** Flag of allocatetion.
**
** gctBOOL Specified
** If user must use this pool, it should set Specified to gcvTRUE,
** otherwise allocator may reserve some memory for other usage, such
** as small block size allocation request.
**
** OUTPUT:
**
** gcuVIDMEM_NODE_PTR * Node
** Pointer to a variable that will hold the allocated memory node.
*/
static gceSTATUS
gckVIDMEM_AllocateLinear(
IN gckKERNEL Kernel,
IN gckVIDMEM Memory,
IN gctSIZE_T Bytes,
IN gctUINT32 Alignment,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
IN gctBOOL Specified,
OUT gcuVIDMEM_NODE_PTR * Node
)
{
gceSTATUS status;
gcuVIDMEM_NODE_PTR node;
gctUINT32 alignment;
gctINT bank, i;
gctBOOL acquired = gcvFALSE;
gctUINT64 mappingInOne = 1;
gcmkHEADER_ARG("Memory=0x%x Bytes=%lu Alignment=%u Type=%d",
Memory, Bytes, Alignment, Type);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Node != gcvNULL);
gcmkVERIFY_ARGUMENT(Type < gcvVIDMEM_TYPE_COUNT);
/* Acquire the mutex. */
gcmkONERROR(gckOS_AcquireMutex(Memory->os, Memory->mutex, gcvINFINITE));
acquired = gcvTRUE;
if (Bytes > Memory->freeBytes)
{
/* Not enough memory. */
status = gcvSTATUS_OUT_OF_MEMORY;
goto OnError;
}
#if gcdSMALL_BLOCK_SIZE
if ((Memory->freeBytes < (Memory->bytes/gcdRATIO_FOR_SMALL_MEMORY))
&& (Bytes >= gcdSMALL_BLOCK_SIZE)
&& (Specified == gcvFALSE)
)
{
/* The left memory is for small memory.*/
status = gcvSTATUS_OUT_OF_MEMORY;
goto OnError;
}
#endif
/* Find the default bank for this surface type. */
gcmkASSERT((gctINT) Type < gcmCOUNTOF(Memory->mapping));
bank = Memory->mapping[Type];
alignment = Alignment;
/* Find a free node in the default bank. */
node = _FindNode(Kernel, Memory, bank, Bytes, Type, &alignment);
/* Out of memory? */
if (node == gcvNULL)
{
/* Walk all lower banks. */
for (i = bank - 1; i >= 0; --i)
{
/* Find a free node inside the current bank. */
node = _FindNode(Kernel, Memory, i, Bytes, Type, &alignment);
if (node != gcvNULL)
{
break;
}
}
}
if (node == gcvNULL)
{
/* Walk all upper banks. */
for (i = bank + 1; i < gcmCOUNTOF(Memory->sentinel); ++i)
{
if (Memory->sentinel[i].VidMem.nextFree == gcvNULL)
{
/* Abort when we reach unused banks. */
break;
}
/* Find a free node inside the current bank. */
node = _FindNode(Kernel, Memory, i, Bytes, Type, &alignment);
if (node != gcvNULL)
{
break;
}
}
}
if (node == gcvNULL)
{
/* Out of memory. */
status = gcvSTATUS_OUT_OF_MEMORY;
goto OnError;
}
/* Do we have an alignment? */
if (alignment > 0)
{
/* Split the node so it is aligned. */
if (_Split(Memory->os, node, alignment))
{
/* Successful split, move to aligned node. */
node = node->VidMem.next;
/* Remove alignment. */
alignment = 0;
}
}
/* Do we have enough memory after the allocation to split it? */
if (node->VidMem.bytes - Bytes > Memory->threshold)
{
/* Adjust the node size. */
_Split(Memory->os, node, Bytes);
}
/* Remove the node from the free list. */
node->VidMem.prevFree->VidMem.nextFree = node->VidMem.nextFree;
node->VidMem.nextFree->VidMem.prevFree = node->VidMem.prevFree;
node->VidMem.nextFree =
node->VidMem.prevFree = gcvNULL;
/* Fill in the information. */
node->VidMem.alignment = alignment;
node->VidMem.parent = Memory;
node->VidMem.logical = gcvNULL;
gcmkONERROR(gckOS_GetProcessID(&node->VidMem.processID));
/* Adjust the number of free bytes. */
Memory->freeBytes -= node->VidMem.bytes;
if (Memory->freeBytes < Memory->minFreeBytes)
{
Memory->minFreeBytes = Memory->freeBytes;
}
gckOS_QueryOption(Memory->os, "allMapInOne", &mappingInOne);
if (!mappingInOne)
{
gcmkONERROR(gckOS_RequestReservedMemory(
Memory->os,
Memory->physicalBase + node->VidMem.offset,
node->VidMem.bytes,
"gal reserved memory",
gcvTRUE,
&node->VidMem.physical
));
}
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex));
/* Return the pointer to the node. */
*Node = node;
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Allocated %u bytes @ 0x%x [0x%08X]",
node->VidMem.bytes, node, node->VidMem.offset);
/* Success. */
gcmkFOOTER_ARG("*Node=0x%x", *Node);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_AllocateVirtual
**
** Construct a new gcuVIDMEM_NODE union for virtual memory.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctSIZE_T Bytes
** Number of byte to allocate.
**
** OUTPUT:
**
** gcuVIDMEM_NODE_PTR * Node
** Pointer to a variable that receives the gcuVIDMEM_NODE union pointer.
*/
static gceSTATUS
gckVIDMEM_AllocateVirtual(
IN gckKERNEL Kernel,
IN gctUINT32 Flag,
IN gctSIZE_T Bytes,
OUT gcuVIDMEM_NODE_PTR * Node
)
{
gckOS os;
gceSTATUS status;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gctPOINTER pointer = gcvNULL;
gctINT i;
gcmkHEADER_ARG("Kernel=0x%x Flag=%x Bytes=%lu", Kernel, Flag, Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Node != gcvNULL);
/* Extract the gckOS object pointer. */
os = Kernel->os;
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
/* Allocate an gcuVIDMEM_NODE union. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer));
node = pointer;
/* Initialize gcuVIDMEM_NODE union for virtual memory. */
node->Virtual.kernel = Kernel;
node->Virtual.contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS;
node->Virtual.logical = gcvNULL;
node->Virtual.kvaddr = gcvNULL;
node->Virtual.bytes = Bytes;
node->Virtual.secure = (Flag & gcvALLOC_FLAG_SECURITY) != 0;
node->Virtual.onFault = (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT) != 0;
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
node->Virtual.lockeds[i] = 0;
node->Virtual.pageTables[i] = gcvNULL;
}
/* Allocate the virtual memory. */
gcmkONERROR(
gckOS_AllocatePagedMemory(os,
Flag,
&node->Virtual.bytes,
&node->Virtual.gid,
&node->Virtual.physical));
/* Calculate required GPU page (4096) count. */
/* Assume start address is 4096 aligned. */
node->Virtual.pageCount = (gctSIZE_T)(((gctUINT64)node->Virtual.bytes + (4096 - 1)) >> 12);
/* Return pointer to the gcuVIDMEM_NODE union. */
*Node = node;
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Created virtual node 0x%x for %u bytes @ 0x%x",
node, Bytes, node->Virtual.physical);
/* Success. */
gcmkFOOTER_ARG("*Node=0x%x", *Node);
return gcvSTATUS_OK;
OnError:
/* Roll back. */
if (node != gcvNULL)
{
/* Free the structure. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
static gceSTATUS
_AddToBlockList(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
VidMemBlock->next = Kernel->vidMemBlock;
Kernel->vidMemBlock = VidMemBlock;
return gcvSTATUS_OK;
}
static gceSTATUS
_RemoveFromBlockList(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
gckVIDMEM_BLOCK vidMemBlock;
gckVIDMEM_BLOCK previous = gcvNULL;
gceHARDWARE_TYPE hwType;
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
for (vidMemBlock = Kernel->vidMemBlock; vidMemBlock != gcvNULL; vidMemBlock = vidMemBlock->next)
{
if (vidMemBlock->addresses[hwType] == VidMemBlock->addresses[hwType])
{
if (previous)
{
previous->next = vidMemBlock->next;
}
else
{
Kernel->vidMemBlock = vidMemBlock->next;
}
vidMemBlock->next = gcvNULL;
break;
}
previous = vidMemBlock;
}
return gcvSTATUS_OK;
}
static gckVIDMEM_BLOCK
_FindFreeBlock(
IN gckKERNEL Kernel,
IN gctSIZE_T Bytes
)
{
gckVIDMEM_BLOCK vidMemBlock;
for (vidMemBlock = Kernel->vidMemBlock; vidMemBlock != gcvNULL; vidMemBlock = vidMemBlock->next)
{
if (vidMemBlock->freeBytes >= Bytes)
{
/* Found the block */
break;
}
}
return vidMemBlock;
}
static gceSTATUS
_SplitVirtualChunk(
IN gckOS Os,
IN gcuVIDMEM_NODE_PTR Node,
IN gctSIZE_T Bytes
)
{
gceSTATUS status = gcvSTATUS_OK;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gctPOINTER pointer;
gctINT i;
if ((Bytes <= 0) || (Bytes > Node->VirtualChunk.bytes))
{
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
/* Allocate a new gcuVIDMEM_NODE object. */
gcmkONERROR(gckOS_Allocate(Os,
gcmSIZEOF(gcuVIDMEM_NODE),
&pointer));
node = pointer;
/* Intialize the new gcuVIDMEM_NODE. */
node->VirtualChunk.offset = Node->VirtualChunk.offset + Bytes;
node->VirtualChunk.bytes = Node->VirtualChunk.bytes - Bytes;
node->VirtualChunk.parent = Node->VirtualChunk.parent;
node->VirtualChunk.kernel = Node->VirtualChunk.kernel;
node->VirtualChunk.kvaddr = gcvNULL;
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
node->VirtualChunk.lockeds[i] = 0;
}
/* Insert chunk behind specified chunk. */
node->VirtualChunk.next = Node->VirtualChunk.next;
node->VirtualChunk.prev = Node;
Node->VirtualChunk.next = node->VirtualChunk.next->VirtualChunk.prev = node;
/* Insert free chunk behind specified chunk. */
node->VirtualChunk.nextFree = Node->VirtualChunk.nextFree;
node->VirtualChunk.prevFree = Node;
Node->VirtualChunk.nextFree = node->VirtualChunk.nextFree->VirtualChunk.prevFree = node;
/* Adjust size of specified chunk. */
Node->VirtualChunk.bytes = Bytes;
OnError:
return status;
}
static gceSTATUS
_MergeVirtualChunk(
IN gckOS Os,
IN gcuVIDMEM_NODE_PTR Node
)
{
gcuVIDMEM_NODE_PTR node;
gceSTATUS status = gcvSTATUS_OK;
node = Node->VirtualChunk.next;
if (Node->VirtualChunk.offset + Node->VirtualChunk.bytes != node->VirtualChunk.offset)
{
/* Corrupted heap. */
gcmkASSERT(
Node->VirtualChunk.offset + Node->VirtualChunk.bytes == node->VirtualChunk.offset);
return gcvSTATUS_HEAP_CORRUPTED;
}
/* Merge. */
Node->VirtualChunk.bytes += node->VirtualChunk.bytes;
/* Unlink next node from linked list. */
Node->VirtualChunk.next = node->VirtualChunk.next;
Node->VirtualChunk.nextFree = node->VirtualChunk.nextFree;
Node->VirtualChunk.next->VirtualChunk.prev =
Node->VirtualChunk.nextFree->VirtualChunk.prevFree = Node;
/* Free the next node. */
status = gcmkOS_SAFE_FREE(Os, node);
return status;
}
static gctBOOL
_IsVidMemBlockFree(
IN gckVIDMEM_BLOCK VidMemBlock
)
{
return (VidMemBlock->freeBytes == VidMemBlock->bytes);
}
static gcuVIDMEM_NODE_PTR
_FindVirtualChunkNode(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock,
IN gctSIZE_T Bytes
)
{
gcuVIDMEM_NODE_PTR node;
if (VidMemBlock->node.VirtualChunk.nextFree == gcvNULL)
{
/* No free chunk left. */
return gcvNULL;
}
for (node = VidMemBlock->node.VirtualChunk.nextFree;
node->VirtualChunk.bytes != 0;
node = node->VirtualChunk.nextFree)
{
if (node->VirtualChunk.bytes >= Bytes)
{
/* Got it. */
return node;
}
}
return gcvNULL;
}
/*******************************************************************************
**
** _ConvertPhysical
**
** Convert CPU physical to GPU address for video node.
**
** INPUT:
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to a gcuVIDMEM_NODE union.
**
** gceCORE Core
** Id of current GPU.
**
** gctPHYS_ADDR_T PhysicalAddress
** CPU physical address
**
** OUTPUT:
** gctUINT32 * Address
** A pointer hold the GPU address.
*/
static gceSTATUS
_ConvertPhysical(
IN gckKERNEL Kernel,
IN gceCORE Core,
IN gcuVIDMEM_NODE_PTR Node,
IN gckVIDMEM_BLOCK VidMemBlock,
IN gctPHYS_ADDR_T PhysicalAddress,
OUT gctUINT32 * Address
)
{
gceSTATUS status;
gctUINT64 physical = 0;
gcmkHEADER_ARG("Node=0x%X", Node);
if ((Node && !Node->Virtual.contiguous) ||
(VidMemBlock && !VidMemBlock->contiguous))
{
/* non-contiguous, mapping is required. */
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
if ((Node && Node->Virtual.secure) ||
(VidMemBlock && VidMemBlock->secure))
{
/* Secure, mapping is forced. */
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
/* Convert to GPU physical address. */
gckOS_CPUPhysicalToGPUPhysical(Kernel->os, PhysicalAddress, &physical);
if ((physical > gcvMAXUINT32) ||
(Node && (physical + Node->Virtual.bytes - 1 > gcvMAXUINT32)) ||
(VidMemBlock && (physical + VidMemBlock->bytes - 1 > gcvMAXUINT32)))
{
/* Above 4G (32bit), mapping is required currently. */
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
if (!gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_MMU))
{
if (physical < Kernel->hardware->baseAddress)
{
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
/* Subtract baseAddress to get a GPU address used for programming. */
physical -= Kernel->hardware->baseAddress;
/* 2G upper is virtual space, better to move to gckHARDWARE section. */
if (physical + Node->Virtual.bytes > 0x80000000)
{
/* End is above 2G, ie virtual space. */
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
*Address = (gctUINT32)physical;
gcmkFOOTER_ARG("*Address=0x%X", *Address);
return gcvSTATUS_OK;
}
else
{
gctBOOL flatMapped;
gcmkONERROR(gckMMU_IsFlatMapped(Kernel->mmu, physical, gcvINVALID_ADDRESS, &flatMapped));
if (!flatMapped)
{
status = gcvSTATUS_NOT_SUPPORTED;
goto OnError;
}
*Address = (gctUINT32)physical;
gcmkFOOTER_ARG("*Address=0x%X", *Address);
return gcvSTATUS_OK;
}
OnError:
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_MapVidMemBlock(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
gceSTATUS status;
gckOS os = Kernel->os;
gctPHYS_ADDR_T physAddr;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p", Kernel, VidMemBlock);
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
gcmkASSERT(VidMemBlock->pageCount > 0);
gcmkONERROR(
gckOS_GetPhysicalFromHandle(os,
VidMemBlock->physical,
0,
&physAddr));
status = _ConvertPhysical(Kernel,
Kernel->core,
gcvNULL,
VidMemBlock,
physAddr,
&VidMemBlock->addresses[hwType]);
if (gcmIS_ERROR(status))
{
/* Allocate pages inside the MMU. */
gcmkONERROR(
gckMMU_AllocatePagesEx(Kernel->mmu,
VidMemBlock->pageCount,
VidMemBlock->type,
gcvPAGE_TYPE_1M,
VidMemBlock->secure,
&VidMemBlock->pageTables[hwType],
&VidMemBlock->addresses[hwType]));
if (VidMemBlock->onFault != gcvTRUE)
{
/* Map the pages. */
gcmkONERROR(
gckOS_Map1MPages(os,
Kernel->core,
VidMemBlock->physical,
VidMemBlock->pageCount,
VidMemBlock->addresses[hwType],
VidMemBlock->pageTables[hwType],
gcvTRUE,
VidMemBlock->type));
}
gcmkONERROR(gckMMU_Flush(Kernel->mmu, VidMemBlock->type));
/* Calculate the GPU virtual address. */
VidMemBlock->addresses[hwType] |= (gctUINT32) (physAddr & ((1 << 20) - 1));
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Mapped video memory block 0x%x to 0x%08X",
VidMemBlock,
VidMemBlock->addresses[hwType]);
return gcvSTATUS_OK;
OnError:
if (VidMemBlock->pageTables[hwType] != gcvNULL)
{
/* Free the pages from the MMU. */
gcmkVERIFY_OK(
gckMMU_FreePages(Kernel->mmu,
VidMemBlock->secure,
gcvPAGE_TYPE_1M,
VidMemBlock->addresses[hwType],
VidMemBlock->pageTables[hwType],
VidMemBlock->pageCount));
VidMemBlock->pageTables[hwType] = gcvNULL;
}
gcmkFOOTER();
return status;
}
static gceSTATUS
_UnmapVidMemBlock(
IN gckMMU Mmu,
IN gceHARDWARE_TYPE HwType,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
gceSTATUS status;
gcmkHEADER_ARG("Mmu=%p VidMemBlock=%p", Mmu, VidMemBlock);
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
if (VidMemBlock->pageTables[HwType] != gcvNULL)
{
/* Free the pages from the MMU. */
gcmkONERROR(
gckMMU_FreePages(Mmu,
VidMemBlock->secure,
gcvPAGE_TYPE_1M,
VidMemBlock->addresses[HwType],
VidMemBlock->pageTables[HwType],
VidMemBlock->pageCount));
VidMemBlock->pageTables[HwType] = gcvNULL;
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_BLOCK_Construct(
IN gckKERNEL Kernel,
IN gctSIZE_T BlockSize,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
OUT gckVIDMEM_BLOCK * VidMemBlock
)
{
gceSTATUS status;
gckVIDMEM_BLOCK vidMemBlock = gcvNULL;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gckOS os = Kernel->os;
gctPOINTER pointer;
gctINT i;
gcmkHEADER_ARG("Kernel=0x%x BlockSize=%lu Type=%x Flag=%x", Kernel, BlockSize, Type);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(BlockSize > 0);
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
/* Allocate an gckVIDMEM_BLOCK object. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_BLOCK), &pointer));
vidMemBlock = pointer;
/* Initialize the gckVIDMEM_BLOCK object. */
vidMemBlock->object.type = gcvOBJ_VIDMEM_BLOCK;
vidMemBlock->os = os;
vidMemBlock->bytes = BlockSize;
vidMemBlock->freeBytes = BlockSize;
/* 1M page count. */
vidMemBlock->pageCount = (gctUINT32)(BlockSize >> 20);
vidMemBlock->type = Type;
vidMemBlock->contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS;
vidMemBlock->secure = (Flag & gcvALLOC_FLAG_SECURITY) != 0;
vidMemBlock->onFault = (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT) != 0;
vidMemBlock->mutex = gcvNULL;
vidMemBlock->physical = gcvNULL;
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
vidMemBlock->pageTables[i] = gcvNULL;
}
/* Allocate the mutex. */
gcmkONERROR(gckOS_CreateMutex(os, &vidMemBlock->mutex));
/* Allocate one gcuVIDMEM_NODE union. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer));
node = pointer;
if (!vidMemBlock->contiguous)
{
Flag |= gcvALLOC_FLAG_1M_PAGES;
}
/* Alloc 1M page size aligned memory block. */
gcmkONERROR(
gckOS_AllocatePagedMemory(os,
Flag,
&BlockSize,
&vidMemBlock->gid,
&vidMemBlock->physical));
/* Map current hardware mmu table with 1M pages for this video memory block. */
gcmkONERROR(gckVIDMEM_MapVidMemBlock(Kernel, vidMemBlock));
/* Initialize gcuVIDMEM_NODE union for virtual memory. */
node->VirtualChunk.kernel = Kernel;
node->VirtualChunk.offset = 0;
node->VirtualChunk.bytes = BlockSize;
node->VirtualChunk.kvaddr = gcvNULL;
node->VirtualChunk.logical = gcvNULL;
node->VirtualChunk.parent = vidMemBlock;
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
node->VirtualChunk.lockeds[i] = 0;
}
/* Initialize the virtual chunk linked-list. */
node->VirtualChunk.next =
node->VirtualChunk.prev =
node->VirtualChunk.nextFree =
node->VirtualChunk.prevFree = &vidMemBlock->node;
vidMemBlock->node.VirtualChunk.next =
vidMemBlock->node.VirtualChunk.prev =
vidMemBlock->node.VirtualChunk.nextFree =
vidMemBlock->node.VirtualChunk.prevFree = node;
vidMemBlock->node.VirtualChunk.bytes = 0;
*VidMemBlock = vidMemBlock;
gcmkFOOTER_ARG("*VidMemBlock=0x%x", *VidMemBlock);
return gcvSTATUS_OK;
OnError:
if (vidMemBlock != gcvNULL)
{
if (vidMemBlock->mutex)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(os, vidMemBlock->mutex));
}
if (vidMemBlock->physical)
{
gcmkVERIFY_OK(gckOS_FreePagedMemory(os,
vidMemBlock->physical,
vidMemBlock->bytes));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, vidMemBlock));
}
if (node != gcvNULL)
{
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node));
}
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_BLOCK_Destroy(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock
)
{
gckDEVICE device = Kernel->device;
gctINT i;
gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p", Kernel, VidMemBlock);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(Kernel != gcvNULL);
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
if (VidMemBlock->physical)
{
gcmkVERIFY_OK(gckOS_FreePagedMemory(Kernel->os,
VidMemBlock->physical,
VidMemBlock->bytes));
}
for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++)
{
if (VidMemBlock->pageTables[i])
{
gcmkVERIFY_OK(_UnmapVidMemBlock(device->mmus[i], i, VidMemBlock));
}
}
/* Free the mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, VidMemBlock->mutex));
/* Free the virtual chunk. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, VidMemBlock->node.VirtualChunk.next));
/* Free the video memory block. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, VidMemBlock));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
static gceSTATUS
_AllocateVirtualChunk(
IN gckKERNEL Kernel,
IN gckVIDMEM_BLOCK VidMemBlock,
IN gceVIDMEM_TYPE Type,
INOUT gctSIZE_T *Bytes,
OUT gcuVIDMEM_NODE_PTR *Node
)
{
gceSTATUS status = gcvSTATUS_OK;
gctBOOL acquired = gcvFALSE;
gcuVIDMEM_NODE_PTR node;
gctSIZE_T bytes;
gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p Type=%x Bytes=%zx",
Kernel, VidMemBlock, Type, *Bytes);
gcmkVERIFY_ARGUMENT(Node != gcvNULL);
gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Type < gcvVIDMEM_TYPE_COUNT);
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, VidMemBlock->mutex, gcvINFINITE));
acquired = gcvTRUE;
bytes = gcmALIGN(*Bytes, 4096);
if (bytes > VidMemBlock->freeBytes)
{
/* No enough memory. */
status = gcvSTATUS_OUT_OF_MEMORY;
goto OnError;
}
node = _FindVirtualChunkNode(Kernel, VidMemBlock, bytes);
if (node == gcvNULL)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
if (node->VirtualChunk.bytes > bytes)
{
/* Split the chunk. */
_SplitVirtualChunk(Kernel->os, node, bytes);
}
/* Remove the chunk from the free list. */
node->VirtualChunk.prevFree->VirtualChunk.nextFree = node->VirtualChunk.nextFree;
node->VirtualChunk.nextFree->VirtualChunk.prevFree = node->VirtualChunk.prevFree;
node->VirtualChunk.nextFree = node->VirtualChunk.prevFree = gcvNULL;
/* Fill in the information. */
node->VirtualChunk.parent = VidMemBlock;
VidMemBlock->freeBytes -= node->VirtualChunk.bytes;
*Bytes = bytes;
*Node = node;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, VidMemBlock->mutex));
}
return status;
}
static gceSTATUS
gckVIDMEM_AllocateVirtualChunk(
IN gckKERNEL Kernel,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
IN gctSIZE_T Bytes,
OUT gcuVIDMEM_NODE_PTR * Node
)
{
gckOS os;
gceSTATUS status;
gcuVIDMEM_NODE_PTR node;
gckVIDMEM_BLOCK vidMemBlock = gcvNULL;
gctSIZE_T blockSize;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%x Flag=%x Bytes=%lu", Kernel, Flag, Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Node != gcvNULL);
/* Extract the gckOS object pointer. */
os = Kernel->os;
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
/* Acquire the vidMem block mutex */
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->vidMemBlockMutex, gcvINFINITE));
acquired = gcvTRUE;
/* Find the free vidmem block. */
vidMemBlock = _FindFreeBlock(Kernel, Bytes);
if (!vidMemBlock)
{
/* Not found, construct new block. */
blockSize = gcmALIGN(Bytes, gcd1M_PAGE_SIZE);
gcmkONERROR(
gckVIDMEM_BLOCK_Construct(Kernel,
blockSize,
Type,
Flag,
&vidMemBlock));
gcmkONERROR(_AddToBlockList(Kernel, vidMemBlock));
}
/* Allocate virtual chunk node in the found block. */
gcmkONERROR(
_AllocateVirtualChunk(Kernel,
vidMemBlock,
Type,
&Bytes,
&node));
/* Return pointer to the gcuVIDMEM_NODE union. */
*Node = node;
/* Release the vidMem block mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex));
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Created virtual node 0x%x for %u bytes @ 0x%x",
node, Bytes, node->Virtual.physical);
/* Success. */
gcmkFOOTER_ARG("*Node=0x%x", *Node);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the vidMem block mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_Free
**
** Free an allocated video memory node.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to a gcuVIDMEM_NODE object.
**
** OUTPUT:
**
** Nothing.
*/
static gceSTATUS
gckVIDMEM_Free(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node
)
{
gceSTATUS status;
gckKERNEL kernel = gcvNULL;
gckVIDMEM memory = gcvNULL;
gckVIDMEM_BLOCK vidMemBlock = gcvNULL;
gcuVIDMEM_NODE_PTR node;
gctBOOL mutexAcquired = gcvFALSE;
gctBOOL vbMutexAcquired = gcvFALSE;
gctBOOL vbListMutexAcquired = gcvFALSE;
gctUINT64 mappingInOne = 1;
gcmkHEADER_ARG("Node=0x%x", Node);
/* Verify the arguments. */
if ((Node == gcvNULL)
|| (Node->VidMem.parent == gcvNULL)
)
{
/* Invalid object. */
gcmkONERROR(gcvSTATUS_INVALID_OBJECT);
}
vidMemBlock = Node->VirtualChunk.parent;
/**************************** Video Memory ********************************/
if (Node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
/* Extract pointer to gckVIDMEM object owning the node. */
memory = Node->VidMem.parent;
/* Acquire the mutex. */
gcmkONERROR(
gckOS_AcquireMutex(memory->os, memory->mutex, gcvINFINITE));
mutexAcquired = gcvTRUE;
if (Node->VidMem.kvaddr)
{
#if gcdCAPTURE_ONLY_MODE
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Node->VidMem.kvaddr));
#else
gcmkONERROR(
gckOS_DestroyKernelMapping(Kernel->os,
Node->VidMem.parent->physical,
Node->VidMem.kvaddr));
#endif
Node->VidMem.kvaddr = gcvNULL;
}
#ifdef __QNXNTO__
/* Unmap the video memory. */
if (Node->VidMem.logical != gcvNULL)
{
gckKERNEL_UnmapVideoMemory(
Kernel,
Node->VidMem.pool,
Node->VidMem.physical,
Node->VidMem.logical,
Node->VidMem.processID,
Node->VidMem.bytes
);
Node->VidMem.logical = gcvNULL;
}
/* Reset. */
Node->VidMem.processID = 0;
/* Don't try to re-free an already freed node. */
if ((Node->VidMem.nextFree == gcvNULL)
&& (Node->VidMem.prevFree == gcvNULL)
)
#endif
{
/* Check if Node is already freed. */
if (Node->VidMem.nextFree)
{
/* Node is alread freed. */
gcmkONERROR(gcvSTATUS_INVALID_DATA);
}
/* Update the number of free bytes. */
memory->freeBytes += Node->VidMem.bytes;
/* Find the next free node. */
for (node = Node->VidMem.next;
node != gcvNULL && node->VidMem.nextFree == gcvNULL;
node = node->VidMem.next) ;
if (node == gcvNULL)
{
gcmkONERROR(gcvSTATUS_INVALID_DATA);
}
/* Insert this node in the free list. */
Node->VidMem.nextFree = node;
Node->VidMem.prevFree = node->VidMem.prevFree;
Node->VidMem.prevFree->VidMem.nextFree =
node->VidMem.prevFree = Node;
/* Is the next node a free node and not the sentinel? */
if ((Node->VidMem.next == Node->VidMem.nextFree)
&& (Node->VidMem.next->VidMem.bytes != 0)
)
{
/* Merge this node with the next node. */
gcmkONERROR(_Merge(memory->os, node = Node));
gcmkASSERT(node->VidMem.nextFree != node);
gcmkASSERT(node->VidMem.prevFree != node);
}
/* Is the previous node a free node and not the sentinel? */
if ((Node->VidMem.prev == Node->VidMem.prevFree)
&& (Node->VidMem.prev->VidMem.bytes != 0)
)
{
/* Merge this node with the previous node. */
gcmkONERROR(_Merge(memory->os, node = Node->VidMem.prev));
gcmkASSERT(node->VidMem.nextFree != node);
gcmkASSERT(node->VidMem.prevFree != node);
}
}
gckOS_QueryOption(memory->os, "allMapInOne", &mappingInOne);
if (!mappingInOne)
{
gckOS_ReleaseReservedMemory(memory->os, Node->VidMem.physical);
Node->VidMem.physical = gcvNULL;
}
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(memory->os, memory->mutex));
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Node 0x%x is freed.",
Node);
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
/* Acquire the vidMem block mutex */
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->vidMemBlockMutex, gcvINFINITE));
vbListMutexAcquired = gcvTRUE;
if (vidMemBlock)
{
gckOS os = vidMemBlock->os;
gcmkONERROR(gckOS_AcquireMutex(os, vidMemBlock->mutex, gcvINFINITE));
vbMutexAcquired = gcvTRUE;
kernel = Node->VirtualChunk.kernel;
if (Node->VirtualChunk.kvaddr)
{
gcmkONERROR(
gckOS_DestroyKernelMapping(kernel->os,
vidMemBlock->physical,
Node->VirtualChunk.kvaddr));
Node->VirtualChunk.kvaddr = gcvNULL;
}
/* Handle the free chunk in the linked-list */
{
/* Check if chunk is in free list. */
if (Node->VirtualChunk.nextFree)
{
/* Chunk is already freed. */
gcmkONERROR(gcvSTATUS_INVALID_DATA);
}
vidMemBlock->freeBytes += Node->VirtualChunk.bytes;
/* Find the next free chunk. */
for (node = Node->VirtualChunk.next;
node != gcvNULL && node->VirtualChunk.nextFree == gcvNULL;
node = node->VirtualChunk.next);
if (node == gcvNULL)
{
gcmkONERROR(gcvSTATUS_INVALID_DATA);
}
/* Insert this chunk in the free list. */
Node->VirtualChunk.nextFree = node;
Node->VirtualChunk.prevFree = node->VirtualChunk.prevFree;
Node->VirtualChunk.prevFree->VirtualChunk.nextFree =
node->VirtualChunk.prevFree = Node;
/* Is the next chunk a free chunk. */
if ((Node->VirtualChunk.next == Node->VirtualChunk.nextFree)
&& (Node->VirtualChunk.next->VirtualChunk.bytes != 0)
)
{
/* Merge this chunk with the next chunk. */
gcmkONERROR(_MergeVirtualChunk(os, node = Node));
gcmkASSERT(node->VirtualChunk.nextFree != node);
gcmkASSERT(node->VirtualChunk.prevFree != node);
}
/* Is the previous chunk a free chunk. */
if ((Node->VirtualChunk.prev == Node->VirtualChunk.prevFree)
&& (Node->VirtualChunk.prev->VirtualChunk.bytes != 0)
)
{
/* Merge this chunk with the previous chunk. */
gcmkONERROR(_MergeVirtualChunk(os, node = Node->VirtualChunk.prev));
gcmkASSERT(node->VirtualChunk.nextFree != node);
gcmkASSERT(node->VirtualChunk.prevFree != node);
}
}
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, vidMemBlock->mutex));
/* Only free the vidmem block when all the chunks are freed. */
if (_IsVidMemBlockFree(vidMemBlock))
{
gcmkONERROR(_RemoveFromBlockList(kernel, vidMemBlock));
gcmkONERROR(gckVIDMEM_BLOCK_Destroy(kernel, vidMemBlock));
}
}
/* Release the vidMem block mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*************************** Virtual Memory *******************************/
/* Get gckKERNEL object. */
kernel = Node->Virtual.kernel;
/* Verify the gckKERNEL object pointer. */
gcmkVERIFY_OBJECT(kernel, gcvOBJ_KERNEL);
if (Node->Virtual.kvaddr)
{
gcmkVERIFY_OK(
gckOS_DestroyKernelMapping(kernel->os,
Node->Virtual.physical,
Node->Virtual.kvaddr));
}
/* Free the virtual memory. */
gcmkVERIFY_OK(gckOS_FreePagedMemory(kernel->os,
Node->Virtual.physical,
Node->Virtual.bytes));
/* Delete the gcuVIDMEM_NODE union. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(kernel->os, Node));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (mutexAcquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(
memory->os, memory->mutex
));
}
if (vbMutexAcquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(
vidMemBlock->os, vidMemBlock->mutex
));
}
if (vbListMutexAcquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(
vidMemBlock->os, Kernel->vidMemBlockMutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_Lock
**
** Lock a video memory node and return its hardware specific address.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to a gcuVIDMEM_NODE union.
**
** OUTPUT:
**
** gctUINT32 * Address
** Pointer to a variable that will hold the hardware specific address.
**
** gctUINT32 * PhysicalAddress
** Pointer to a variable that will hold the bus address of a contiguous
** video node.
*/
static gceSTATUS
gckVIDMEM_Lock(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
OUT gctUINT32 * Address
)
{
gckOS os;
gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node);
/* Extract the gckOS object pointer. */
os = Kernel->os;
/* Increment the lock count. */
if (Node->VidMem.locked++ == 0)
{
gctUINT32 address;
gctUINT32 offset = (gctUINT32)Node->VidMem.offset;
switch (Node->VidMem.pool)
{
case gcvPOOL_LOCAL_EXTERNAL:
address = Kernel->externalBaseAddress + offset;
break;
case gcvPOOL_LOCAL_INTERNAL:
address = Kernel->internalBaseAddress + offset;
break;
case gcvPOOL_INTERNAL_SRAM:
address = Kernel->sRAMBaseAddresses[Kernel->sRAMIndex] + offset;
break;
case gcvPOOL_EXTERNAL_SRAM:
address = Kernel->extSRAMBaseAddresses[Kernel->extSRAMIndex] + offset;
break;
default:
gcmkASSERT(Node->VidMem.pool == gcvPOOL_SYSTEM);
/*FALLTHRU*/
case gcvPOOL_SYSTEM:
address = Kernel->contiguousBaseAddress + offset;
break;
}
/* Save address. */
Node->VidMem.address = address;
}
*Address = Node->VidMem.address;
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Locked node 0x%x (%d) @ 0x%08X",
Node,
Node->VidMem.locked,
*Address);
gcmkFOOTER_ARG("*Address=0x%08X", *Address);
return gcvSTATUS_OK;
}
static gceSTATUS
gckVIDMEM_LockVirtual(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
OUT gctUINT32 * Address
)
{
gceSTATUS status;
gctPHYS_ADDR_T physicalAddress;
gctBOOL locked = gcvFALSE;
gckOS os = Kernel->os;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node);
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
gcmkONERROR(
gckOS_GetPhysicalFromHandle(os,
Node->Virtual.physical,
0,
&physicalAddress));
/* Increment the lock count. */
if (Node->Virtual.lockeds[hwType]++ == 0)
{
locked = gcvTRUE;
status = _ConvertPhysical(
Kernel,
Kernel->core,
Node,
gcvNULL,
physicalAddress,
&Node->Virtual.addresses[hwType]
);
if (gcmIS_ERROR(status))
{
/* Do GPU address mapping. */
#if gcdSECURITY
gctPHYS_ADDR physicalArrayPhysical;
gctPOINTER physicalArrayLogical;
gcmkONERROR(gckOS_AllocatePageArray(
os,
Node->Virtual.physical,
Node->Virtual.pageCount,
&physicalArrayLogical,
&physicalArrayPhysical
));
gcmkONERROR(gckKERNEL_SecurityMapMemory(
Kernel,
physicalArrayLogical,
Node->Virtual.pageCount,
&Node->Virtual.addresses[hwType]
));
gcmkONERROR(gckOS_FreeNonPagedMemory(
os,
physicalArrayPhysical,
physicalArrayLogical,
1
));
#else
{
/* Allocate pages inside the MMU. */
gcmkONERROR(
gckMMU_AllocatePagesEx(Kernel->mmu,
Node->Virtual.pageCount,
Node->Virtual.type,
gcvPAGE_TYPE_4K,
Node->Virtual.secure,
&Node->Virtual.pageTables[hwType],
&Node->Virtual.addresses[hwType]));
}
if (Node->Virtual.onFault != gcvTRUE)
{
#if gcdENABLE_TRUST_APPLICATION
if (Kernel->hardware->options.secureMode == gcvSECURE_IN_TA)
{
gcmkONERROR(gckKERNEL_MapInTrustApplicaiton(
Kernel,
Node->Virtual.logical,
Node->Virtual.physical,
Node->Virtual.addresses[hwType],
Node->Virtual.pageCount
));
}
else
#endif
{
gcmkDUMP(os, "#[mmu: dynamic mapping: address=0x%08X pageCount=%lu]",
Node->Virtual.addresses[hwType],
(unsigned long)Node->Virtual.pageCount);
/* Map the pages. */
gcmkONERROR(gckOS_MapPagesEx(os,
Kernel->core,
Node->Virtual.physical,
Node->Virtual.pageCount,
Node->Virtual.addresses[hwType],
Node->Virtual.pageTables[hwType],
gcvTRUE,
Node->Virtual.type));
}
}
{
gcmkONERROR(gckMMU_Flush(Kernel->mmu, Node->Virtual.type));
}
#endif
/* GPU MMU page size is fixed at 4096 now. */
Node->Virtual.addresses[hwType] |= (gctUINT32)physicalAddress & (4096 - 1);
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Mapped virtual node 0x%x to 0x%08X",
Node,
Node->Virtual.addresses[hwType]);
}
/* Return hardware address. */
*Address = Node->Virtual.addresses[hwType];
gcmkFOOTER_ARG("*Address=0x%08X", *Address);
return gcvSTATUS_OK;
OnError:
if (locked)
{
if (Node->Virtual.pageTables[hwType] != gcvNULL)
{
{
/* Free the pages from the MMU. */
gcmkVERIFY_OK(
gckMMU_FreePages(Kernel->mmu,
Node->Virtual.secure,
gcvPAGE_TYPE_4K,
Node->Virtual.addresses[hwType],
Node->Virtual.pageTables[hwType],
Node->Virtual.pageCount));
}
Node->Virtual.pageTables[hwType] = gcvNULL;
}
Node->Virtual.lockeds[hwType]--;
}
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_LockVirtualChunk(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
OUT gctUINT32 * Address
)
{
gceSTATUS status = gcvSTATUS_OK;
gckVIDMEM_BLOCK vidMemBlock = Node->VirtualChunk.parent;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node);
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
gcmkASSERT(vidMemBlock != gcvNULL);
/* Increment the lock count. */
if (Node->VirtualChunk.lockeds[hwType]++ == 0)
{
if (!vidMemBlock->pageTables[hwType])
{
/* Map current hardware mmu table with 1M pages for this video memory block. */
gcmkONERROR(gckVIDMEM_MapVidMemBlock(Kernel, vidMemBlock));
}
Node->VirtualChunk.addresses[hwType] = vidMemBlock->addresses[hwType]
+ (gctUINT32)Node->VirtualChunk.offset;
}
/* Return hardware address. */
*Address = Node->VirtualChunk.addresses[hwType];
gcmkFOOTER_ARG("*Address=0x%08X", *Address);
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_Unlock
**
** Unlock a video memory node.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gcuVIDMEM_NODE_PTR Node
** Pointer to a locked gcuVIDMEM_NODE union.
**
** gctBOOL * Asynchroneous
** Pointer to a variable specifying whether the surface should be
** unlocked asynchroneously or not.
**
** OUTPUT:
**
** gctBOOL * Asynchroneous
** Pointer to a variable receiving the number of bytes used in the
** command buffer specified by 'Commands'. If gcvNULL, there is no
** command buffer.
*/
static gceSTATUS
gckVIDMEM_Unlock(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
IN OUT gctBOOL * Asynchroneous
)
{
gceSTATUS status;
gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d",
Node, gcmOPT_VALUE(Asynchroneous));
if (Node->VidMem.locked <= 0)
{
/* The surface was not locked. */
gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED);
}
if (Asynchroneous != gcvNULL)
{
/* Schedule an event to sync with GPU. */
*Asynchroneous = gcvTRUE;
}
else
{
/* Decrement the lock count. */
Node->VidMem.locked--;
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Unlocked node %p (%d)",
Node,
Node->VidMem.locked);
/* Success. */
gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous));
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_UnlockVirtual(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
IN OUT gctBOOL * Asynchroneous
)
{
gceSTATUS status;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d",
Node, gcmOPT_VALUE(Asynchroneous));
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
if (Asynchroneous != gcvNULL)
{
/* Schedule the surface to be unlocked. */
*Asynchroneous = gcvTRUE;
}
else
{
if (Node->Virtual.lockeds[hwType] == 0)
{
gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED);
}
/* Decrement lock count. */
--Node->Virtual.lockeds[hwType];
/* See if we can unlock the resources. */
if (Node->Virtual.lockeds[hwType] == 0)
{
gctUINT32 address;
/* Adjust address to page aligned for underlying functions. */
address = Node->Virtual.addresses[hwType] & ~(4096 - 1);
#if gcdSECURITY
if (Node->Virtual.addresses[hwType] > 0x80000000)
{
gcmkONERROR(gckKERNEL_SecurityUnmapMemory(
Kernel,
address,
Node->Virtual.pageCount
));
}
#else
/* Free the page table. */
if (Node->Virtual.pageTables[hwType] != gcvNULL)
{
{
gcmkONERROR(
gckMMU_FreePages(Kernel->mmu,
Node->Virtual.secure,
gcvPAGE_TYPE_4K,
address,
Node->Virtual.pageTables[hwType],
Node->Virtual.pageCount));
}
gcmkONERROR(gckOS_UnmapPages(
Kernel->os,
Node->Virtual.pageCount,
address
));
/* Mark page table as freed. */
Node->Virtual.pageTables[hwType] = gcvNULL;
}
#endif
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Unmapped virtual node %p from 0x%08X",
Node, Node->Virtual.addresses[hwType]);
}
/* Success. */
gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous));
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_UnlockVirtualChunk(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR Node,
IN OUT gctBOOL * Asynchroneous
)
{
gceSTATUS status;
gceHARDWARE_TYPE hwType;
gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d",
Node, gcmOPT_VALUE(Asynchroneous));
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
if (Asynchroneous != gcvNULL)
{
/* Schedule an event to sync with GPU. */
*Asynchroneous = gcvTRUE;
}
else
{
if (Node->VirtualChunk.lockeds[hwType] == 0)
{
/* The surface was not locked. */
gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED);
}
/* Unmap and free pages when video memory free. */
/* Decrement the lock count. */
--Node->VirtualChunk.lockeds[hwType];
}
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM,
"Unlocked node %p (%d)",
Node,
Node->VirtualChunk.lockeds[hwType]);
/* Success. */
gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous));
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckVIDMEM_HANDLE_Allocate
**
** Allocate a handle for a gckVIDMEM_NODE object.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gckVIDMEM_NODE Node
** Pointer to a gckVIDMEM_NODE object.
**
** OUTPUT:
**
** gctUINT32 * Handle
** Pointer to a variable receiving a handle represent this
** gckVIDMEM_NODE in userspace.
*/
gceSTATUS
gckVIDMEM_HANDLE_Allocate(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE Node,
OUT gctUINT32 * Handle
)
{
gceSTATUS status;
gctUINT32 processID = 0;
gctPOINTER pointer = gcvNULL;
gctPOINTER handleDatabase = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctUINT32 handle = 0;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gckOS os = Kernel->os;
gcmkHEADER_ARG("Kernel=0x%X, Node=0x%X", Kernel, Node);
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
/* Allocate a gckVIDMEM_HANDLE object. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_HANDLE), &pointer));
gcmkVERIFY_OK(gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsVIDMEM_HANDLE)));
handleObject = pointer;
gcmkONERROR(gckOS_AtomConstruct(os, &handleObject->reference));
/* Set default reference count to 1. */
gckOS_AtomSet(os, handleObject->reference, 1);
gcmkVERIFY_OK(gckOS_GetProcessID(&processID));
gcmkONERROR(
gckKERNEL_FindHandleDatbase(Kernel,
processID,
&handleDatabase,
&mutex));
/* Allocate a handle for this object. */
gcmkONERROR(
gckKERNEL_AllocateIntegerId(handleDatabase, handleObject, &handle));
handleObject->node = Node;
handleObject->handle = handle;
*Handle = handle;
gcmkFOOTER_ARG("*Handle=%d", *Handle);
return gcvSTATUS_OK;
OnError:
if (handleObject != gcvNULL)
{
if (handleObject->reference != gcvNULL)
{
gcmkVERIFY_OK(gckOS_AtomDestroy(os, handleObject->reference));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, handleObject));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_HANDLE_Reference(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Handle
)
{
gceSTATUS status;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gctPOINTER database = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctINT32 oldValue = 0;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Handle=%d PrcoessID=%d", Handle, ProcessID);
gcmkONERROR(
gckKERNEL_FindHandleDatbase(Kernel, ProcessID, &database, &mutex));
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
/* Translate handle to gckVIDMEM_HANDLE object. */
gcmkONERROR(
gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject));
/* Increase the reference count. */
gckOS_AtomIncrement(Kernel->os, handleObject->reference, &oldValue);
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_HANDLE_Dereference(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Handle
)
{
gceSTATUS status;
gctPOINTER handleDatabase = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctINT32 oldValue = 0;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Handle=%d PrcoessID=%d", Handle, ProcessID);
gcmkONERROR(
gckKERNEL_FindHandleDatbase(Kernel,
ProcessID,
&handleDatabase,
&mutex));
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
/* Translate handle to gckVIDMEM_HANDLE. */
gcmkONERROR(
gckKERNEL_QueryIntegerId(handleDatabase, Handle, (gctPOINTER *)&handleObject));
gckOS_AtomDecrement(Kernel->os, handleObject->reference, &oldValue);
if (oldValue == 1)
{
/* Remove handle from database if this is the last reference. */
gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(handleDatabase, Handle));
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
if (oldValue == 1)
{
gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, handleObject->reference));
gcmkOS_SAFE_FREE(Kernel->os, handleObject);
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_HANDLE_Lookup(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Handle,
OUT gckVIDMEM_NODE * Node
)
{
gceSTATUS status;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gckVIDMEM_NODE node = gcvNULL;
gctPOINTER database = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X ProcessID=%d Handle=%d",
Kernel, ProcessID, Handle);
gcmkONERROR(
gckKERNEL_FindHandleDatbase(Kernel, ProcessID, &database, &mutex));
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
gcmkONERROR(
gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject));
node = handleObject->node;
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
*Node = node;
gcmkFOOTER_ARG("*Node=%d", *Node);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_HANDLE_Lookup2(
IN gckKERNEL Kernel,
IN gcsDATABASE_PTR Database,
IN gctUINT32 Handle,
OUT gckVIDMEM_NODE * Node
)
{
gceSTATUS status;
gckVIDMEM_HANDLE handleObject = gcvNULL;
gckVIDMEM_NODE node = gcvNULL;
gctPOINTER database = gcvNULL;
gctPOINTER mutex = gcvNULL;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X Database=%p Handle=%d",
Kernel, Database, Handle);
database = Database->handleDatabase;
mutex = Database->handleDatabaseMutex;
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
gcmkONERROR(
gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject));
node = handleObject->node;
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
*Node = node;
gcmkFOOTER_ARG("*Node=%d", *Node);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
static gceSTATUS
gckVIDMEM_NODE_Construct(
IN gckKERNEL Kernel,
IN gcuVIDMEM_NODE_PTR VideoNode,
IN gceVIDMEM_TYPE Type,
IN gcePOOL Pool,
OUT gckVIDMEM_NODE * NodeObject
)
{
gceSTATUS status;
gckVIDMEM_NODE node = gcvNULL;
gctPOINTER pointer = gcvNULL;
gckOS os = Kernel->os;
gctUINT i;
/* Construct a node. */
gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_NODE), &pointer));
gcmkVERIFY_OK(gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsVIDMEM_NODE)));
node = pointer;
node->metadata.magic = VIV_VIDMEM_METADATA_MAGIC;
node->metadata.ts_fd = -1;
#ifdef gcdANDROID
node->metadata.ts_address = 0;
#endif
node->node = VideoNode;
node->kernel = Kernel;
node->type = Type;
node->pool = Pool;
gcmkONERROR(gckOS_AtomConstruct(os, &node->reference));
gcmkONERROR(gckOS_CreateMutex(os, &node->mutex));
for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
{
gcmkONERROR(gckOS_CreateSignal(os, gcvFALSE, &node->sync[i].signal));
}
/* Reference is 1 by default . */
gckOS_AtomSet(os, node->reference, 1);
gcmkVERIFY_OK(
gckOS_AcquireMutex(Kernel->os,
Kernel->db->videoMemListMutex,
gcvINFINITE));
/* Add into video memory node list. */
gcsLIST_Add(&node->link, &Kernel->db->videoMemList);
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex));
*NodeObject = node;
return gcvSTATUS_OK;
OnError:
if (node != gcvNULL)
{
if (node->mutex)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(os, node->mutex));
}
if (node->reference != gcvNULL)
{
gcmkVERIFY_OK(gckOS_AtomDestroy(os, node->reference));
}
for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
{
if (node->sync[i].signal != gcvNULL)
{
gcmkVERIFY_OK(gckOS_DestroySignal(os, node->sync[i].signal));
}
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node));
}
return status;
}
gceSTATUS
gckVIDMEM_NODE_AllocateLinear(
IN gckKERNEL Kernel,
IN gckVIDMEM VideoMemory,
IN gcePOOL Pool,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
IN gctUINT32 Alignment,
IN gctBOOL Specified,
IN OUT gctSIZE_T * Bytes,
OUT gckVIDMEM_NODE * NodeObject
)
{
gceSTATUS status;
gctSIZE_T bytes = *Bytes;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gckVIDMEM_NODE nodeObject = gcvNULL;
gcmkHEADER_ARG("Kernel=%p VideoMemory=%p Pool=%d Alignment=%d Type=%d *Bytes=%u",
Kernel, VideoMemory, Pool, Alignment, Type, bytes);
gcmkONERROR(
gckVIDMEM_AllocateLinear(Kernel,
VideoMemory,
bytes,
Alignment,
Type,
Flag,
Specified,
&node));
/* Update pool. */
node->VidMem.pool = Pool;
bytes = node->VidMem.bytes;
/* Construct a node. */
gcmkONERROR(
gckVIDMEM_NODE_Construct(Kernel, node, Type, Pool, &nodeObject));
*Bytes = bytes;
*NodeObject = nodeObject;
gcmkFOOTER_ARG("*Bytes=%u *NodeObject=%p", bytes, nodeObject);
return gcvSTATUS_OK;
OnError:
if (node)
{
gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_AllocateVirtual(
IN gckKERNEL Kernel,
IN gcePOOL Pool,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
IN OUT gctSIZE_T * Bytes,
OUT gckVIDMEM_NODE * NodeObject
)
{
gceSTATUS status;
gctSIZE_T bytes = *Bytes;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gckVIDMEM_NODE nodeObject = gcvNULL;
gcmkHEADER_ARG("Kernel=%p Pool=%d Type=%d Flag=%x *Bytes=%u",
Kernel, Pool, Type, Flag, bytes);
gcmkONERROR(
gckVIDMEM_AllocateVirtual(Kernel, Flag, bytes, &node));
/* Update type. */
node->Virtual.type = Type;
bytes = node->Virtual.bytes;
/* Construct a node. */
gcmkONERROR(
gckVIDMEM_NODE_Construct(Kernel, node, Type, Pool, &nodeObject));
*Bytes = bytes;
*NodeObject = nodeObject;
gcmkFOOTER_ARG("*Bytes=%u *NodeObject=%p", bytes, nodeObject);
return gcvSTATUS_OK;
OnError:
if (node)
{
gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_AllocateVirtualChunk(
IN gckKERNEL Kernel,
IN gcePOOL Pool,
IN gceVIDMEM_TYPE Type,
IN gctUINT32 Flag,
IN OUT gctSIZE_T * Bytes,
OUT gckVIDMEM_NODE * NodeObject
)
{
gceSTATUS status;
gctSIZE_T bytes = *Bytes;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gckVIDMEM_NODE nodeObject = gcvNULL;
gcmkHEADER_ARG("Kernel=%p Pool=%d Type=%d Flag=%x *Bytes=%u",
Kernel, Pool, Type, Flag, bytes);
gcmkONERROR(
gckVIDMEM_AllocateVirtualChunk(Kernel, Type, Flag, bytes, &node));
bytes = node->VirtualChunk.bytes;
/* Construct a node. */
gcmkONERROR(
gckVIDMEM_NODE_Construct(Kernel, node, Type, Pool, &nodeObject));
*Bytes = bytes;
*NodeObject = nodeObject;
gcmkFOOTER_ARG("*Bytes=%u *NodeObject=%p", bytes, nodeObject);
return gcvSTATUS_OK;
OnError:
if (node)
{
gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_Reference(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject
)
{
gctINT32 oldValue;
gcmkHEADER_ARG("Kernel=0x%X NodeObject=0x%X", Kernel, NodeObject);
gcmkVERIFY_ARGUMENT(NodeObject != gcvNULL);
gckOS_AtomIncrement(Kernel->os, NodeObject->reference, &oldValue);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckVIDMEM_NODE_Dereference(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject
)
{
gctINT32 oldValue = 0;
gctPOINTER database = Kernel->db->nameDatabase;
gctPOINTER mutex = Kernel->db->nameDatabaseMutex;
gctUINT i;
gcmkHEADER_ARG("Kernel=0x%X NodeObject=0x%X", Kernel, NodeObject);
gcmkVERIFY_ARGUMENT(NodeObject != gcvNULL);
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
gcmkVERIFY_OK(gckOS_AtomDecrement(Kernel->os, NodeObject->reference, &oldValue));
if (oldValue == 1 && NodeObject->name)
{
/* Free name if exists. */
gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(database, NodeObject->name));
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
if (oldValue == 1)
{
gcmkVERIFY_OK(
gckOS_AcquireMutex(Kernel->os,
Kernel->db->videoMemListMutex,
gcvINFINITE));
/* Remove from video memory node list. */
gcsLIST_Del(&NodeObject->link);
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex));
/* Free gcuVIDMEM_NODE. */
gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, NodeObject->node));
gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, NodeObject->reference));
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, NodeObject->mutex));
for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
{
if (NodeObject->sync[i].signal != gcvNULL)
{
gcmkVERIFY_OK(gckOS_DestroySignal(Kernel->os, NodeObject->sync[i].signal));
}
}
/* Should not cause recursive call since tsNode->tsNode should be NULL */
if (NodeObject->tsNode)
{
gcmkASSERT(!NodeObject->tsNode->tsNode);
gckVIDMEM_NODE_Dereference(Kernel, NodeObject->tsNode);
}
gcmkOS_SAFE_FREE(Kernel->os, NodeObject);
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckVIDMEM_NODE_GetReference(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
OUT gctINT32 * ReferenceCount
)
{
gctINT32 value;
gckOS_AtomGet(Kernel->os, NodeObject->reference, &value);
*ReferenceCount = value;
return gcvSTATUS_OK;
}
gceSTATUS
gckVIDMEM_NODE_Lock(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
OUT gctUINT32 * Address
)
{
gceSTATUS status;
gckOS os = Kernel->os;
gctBOOL acquired = gcvFALSE;
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gcmkHEADER_ARG("NodeObject=%p", NodeObject);
/* Grab the mutex. */
gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE));
acquired = gcvTRUE;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
gcmkONERROR(gckVIDMEM_Lock(Kernel, node, Address));
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
gcmkONERROR(gckVIDMEM_LockVirtualChunk(Kernel, node, Address));
}
else
{
gcmkONERROR(gckVIDMEM_LockVirtual(Kernel, node, Address));
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
gcmkFOOTER_ARG("*Address=0x%08X", *Address);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_Unlock(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
IN gctUINT32 ProcessID,
IN OUT gctBOOL * Asynchroneous
)
{
gceSTATUS status;
gckOS os = Kernel->os;
gctBOOL acquired = gcvFALSE;
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gcmkHEADER_ARG("NodeObject=%p Asynchroneous=%p", NodeObject, Asynchroneous);
/* Grab the mutex. */
gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE));
acquired = gcvTRUE;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
gcmkONERROR(gckVIDMEM_Unlock(Kernel, node, Asynchroneous));
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
gcmkONERROR(gckVIDMEM_UnlockVirtualChunk(Kernel, node, Asynchroneous));
}
else
{
gcmkONERROR(gckVIDMEM_UnlockVirtual(Kernel, node, Asynchroneous));
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
gcmkFOOTER_ARG("*Asynchroneous=0x%08X", gcmOPT_VALUE(Asynchroneous));
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_CleanCache(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
IN gctSIZE_T Offset,
IN gctPOINTER Logical,
IN gctSIZE_T Bytes
)
{
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gctPHYS_ADDR physHandle = gcvNULL;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gceSTATUS status;
gcmkHEADER_ARG("Kernel=%p NodeObject=%d Offset=0x%llx Logical=%p Bytes=0x%llx",
Kernel, NodeObject, Offset, Logical, Bytes);
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
gcmkONERROR(gckOS_MemoryBarrier(Kernel->os, Logical));
/* Reserved pool can't be cacheable */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
physHandle = vidMemBlock->physical;
}
else
{
physHandle = node->Virtual.physical;
}
gcmkONERROR(gckOS_CacheFlush(
Kernel->os,
0,
physHandle,
Offset,
Logical,
Bytes
));
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_InvalidateCache(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
IN gctSIZE_T Offset,
IN gctPOINTER Logical,
IN gctSIZE_T Bytes
)
{
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gctPHYS_ADDR physHandle = gcvNULL;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gceSTATUS status;
gcmkHEADER_ARG("Kernel=%p NodeObject=%d Offset=0x%llx Logical=%p Bytes=0x%llx",
Kernel, NodeObject, Offset, Logical, Bytes);
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
/* Reserved pool can't be cacheable */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
physHandle = vidMemBlock->physical;
}
else
{
physHandle = node->Virtual.physical;
}
gcmkONERROR(gckOS_CacheInvalidate(
Kernel->os,
0,
physHandle,
Offset,
Logical,
Bytes
));
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_GetLockCount(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
OUT gctINT32 * LockCount
)
{
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gctINT32 lockCount = 0;
gctINT i = 0;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
lockCount = node->VidMem.locked;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
for (; i < gcvHARDWARE_NUM_TYPES; i++)
{
lockCount += node->VirtualChunk.lockeds[i];
}
}
else
{
for (; i < gcvHARDWARE_NUM_TYPES; i++)
{
lockCount += node->Virtual.lockeds[i];
}
}
*LockCount = lockCount;
return gcvSTATUS_OK;
}
gceSTATUS
gckVIDMEM_NODE_LockCPU(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
IN gctBOOL Cacheable,
IN gctBOOL FromUser,
OUT gctPOINTER * Logical
)
{
gceSTATUS status;
gckOS os = Kernel->os;
gctBOOL acquired = gcvFALSE;
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gctPOINTER logical = gcvNULL;
gcmkHEADER_ARG("NodeObject=%p", NodeObject);
/* Grab the mutex. */
gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE));
acquired = gcvTRUE;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
if (Cacheable == gcvTRUE)
{
gcmkONERROR(gcvSTATUS_INVALID_REQUEST);
}
if (FromUser)
{
#if gcdCAPTURE_ONLY_MODE
node->VidMem.logical = NodeObject->captureLogical;
#else
/* Map video memory pool to user space. */
gcmkONERROR(
gckKERNEL_MapVideoMemory(Kernel,
gcvTRUE,
node->VidMem.pool,
node->VidMem.physical,
(gctUINT32)node->VidMem.offset,
(gctUINT32)node->VidMem.bytes,
&node->VidMem.logical));
#endif
logical = node->VidMem.logical;
}
else
{
/* Map video memory pool to kernel space. */
if (!node->VidMem.kvaddr)
{
#if gcdCAPTURE_ONLY_MODE
gcmkONERROR(gckOS_Allocate(os,
node->VidMem.bytes,
&node->VidMem.kvaddr));
#else
gcmkONERROR(
gckOS_CreateKernelMapping(os,
node->VidMem.parent->physical,
node->VidMem.offset,
node->VidMem.bytes,
&node->VidMem.kvaddr));
#endif
}
logical = node->VidMem.kvaddr;
}
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
if (FromUser)
{
/* Lock the entire vidmem block. */
gcmkONERROR(
gckOS_LockPages(os,
vidMemBlock->physical,
vidMemBlock->bytes,
Cacheable,
&logical));
/* Get the logical with offset in block. */
logical = (uint8_t *)logical + node->VirtualChunk.offset;
node->VirtualChunk.logical = logical;
}
else
{
/* Map once and will cancel map when free. */
if (!node->VirtualChunk.kvaddr)
{
gcmkONERROR(
gckOS_CreateKernelMapping(os,
vidMemBlock->physical,
node->VirtualChunk.offset,
node->VirtualChunk.bytes,
&node->VirtualChunk.kvaddr));
}
logical = node->VirtualChunk.kvaddr;
}
}
else
{
if (FromUser)
{
gcmkONERROR(
gckOS_LockPages(os,
node->Virtual.physical,
node->Virtual.bytes,
Cacheable,
&logical));
node->Virtual.logical = logical;
}
else
{
/* Map once and will cancel map when free. */
if (!node->Virtual.kvaddr)
{
gcmkONERROR(
gckOS_CreateKernelMapping(os,
node->Virtual.physical,
0,
node->Virtual.bytes,
&node->Virtual.kvaddr));
}
logical = node->Virtual.kvaddr;
}
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
*Logical = logical;
gcmkFOOTER_ARG("*Logical=%p", logical);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_UnlockCPU(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
IN gctUINT32 ProcessID,
IN gctBOOL FromUser,
IN gctBOOL Defer
)
{
gceSTATUS status;
gckOS os = Kernel->os;
gctBOOL acquired = gcvFALSE;
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gcmkHEADER_ARG("NodeObject=%p", NodeObject);
/* Grab the mutex. */
gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE));
acquired = gcvTRUE;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
if (FromUser)
{
#if gcdCAPTURE_ONLY_MODE || defined __QNXNTO__
/* Do nothing here. */
#else
if (!Defer)
{
/* Unmap the video memory. */
if (node->VidMem.logical != gcvNULL)
{
gckKERNEL_UnmapVideoMemory(
Kernel,
node->VidMem.pool,
node->VidMem.physical,
node->VidMem.logical,
node->VidMem.processID,
node->VidMem.bytes
);
node->VidMem.logical = gcvNULL;
}
/* Reset. */
node->VidMem.processID = 0;
}
#endif
}
else
{
/*
* Kernel side may lock for CPU access for multiple times. Since
* we don't have lock counts currently, we don't cancel CPU
* mapping here, and will cancel at 'free' instead.
*/
/*
gcmkONERROR(
gckOS_DestroyKernelMapping(os,
node->VidMem.parent->physical,
node->VidMem.kvaddr));
node->VidMem.kvaddr = gcvNULL;
*/
}
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
if (FromUser)
{
gcmkONERROR(
gckOS_UnlockPages(os,
vidMemBlock->physical,
vidMemBlock->bytes,
node->VirtualChunk.logical));
}
else
{
/* Nothing to do. */
}
}
else
{
if (FromUser)
{
gcmkONERROR(
gckOS_UnlockPages(os,
node->Virtual.physical,
node->Virtual.bytes,
node->Virtual.logical));
}
else
{
/*
* Kernel side may lock for CPU access for multiple times. Since
* we don't have lock counts currently, we don't cancel CPU
* mapping here, and will cancel at 'free' instead.
*/
/*
gcmkONERROR(
gckOS_DestroyKernelMapping(os,
node->Virtual.physical,
node->Virtual.kvaddr));
node->Virtual.kvaddr = gcvNULL;
*/
}
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_GetPhysical(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
IN gctUINT32 Offset,
OUT gctPHYS_ADDR_T * PhysicalAddress
)
{
gceSTATUS status;
gckOS os = Kernel->os;
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gcmkHEADER_ARG("NodeObject=%p", NodeObject);
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
if (Offset >= node->VidMem.bytes)
{
/* Exceeds node size. */
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
*PhysicalAddress = node->VidMem.parent->physicalBase
+ node->VidMem.offset
+ Offset;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
if (Offset >= node->VirtualChunk.bytes)
{
/* Exceeds node size. */
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
gcmkONERROR(
gckOS_GetPhysicalFromHandle(os,
vidMemBlock->physical,
(gctUINT32)node->VirtualChunk.offset + Offset,
PhysicalAddress));
}
else
{
if (Offset >= node->Virtual.bytes)
{
/* Exceeds node size. */
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
gcmkONERROR(
gckOS_GetPhysicalFromHandle(os,
node->Virtual.physical,
Offset,
PhysicalAddress));
}
gcmkFOOTER_ARG("*PhysicalAddress=0x%llx", *PhysicalAddress);
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_GetGid(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
OUT gctUINT32 * Gid
)
{
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
*Gid = 0;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
*Gid = vidMemBlock->gid;
}
else
{
*Gid = node->Virtual.gid;
}
return gcvSTATUS_OK;
}
gceSTATUS
gckVIDMEM_NODE_GetSize(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
OUT gctSIZE_T * Size
)
{
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
*Size = node->VidMem.bytes;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
*Size = node->VirtualChunk.bytes;
}
else
{
*Size = node->Virtual.bytes;
}
return gcvSTATUS_OK;
}
gceSTATUS
gckVIDMEM_NODE_GetType(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
OUT gceVIDMEM_TYPE * Type,
OUT gcePOOL * Pool
)
{
if (Type)
{
*Type = NodeObject->type;
}
if (Pool)
{
*Pool = NodeObject->pool;
}
return gcvSTATUS_OK;
}
#if defined(CONFIG_DMA_SHARED_BUFFER)
/*******************************************************************************
**
**
** Code for dma_buf ops
**
**
*******************************************************************************/
#include <linux/slab.h>
#include <linux/mm_types.h>
#include <linux/dma-buf.h>
static struct sg_table *_dmabuf_map(struct dma_buf_attachment *attachment,
enum dma_data_direction direction)
{
struct sg_table *sgt = gcvNULL;
struct dma_buf *dmabuf = attachment->dmabuf;
gckVIDMEM_NODE nodeObject = dmabuf->priv;
gceSTATUS status = gcvSTATUS_OK;
do
{
gcuVIDMEM_NODE_PTR node = nodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gctPHYS_ADDR physical = gcvNULL;
gctSIZE_T offset = 0;
gctSIZE_T bytes = 0;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
physical = node->VidMem.parent->physical;
offset = node->VidMem.offset;
bytes = node->VidMem.bytes;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
physical = vidMemBlock->physical;
offset = node->VirtualChunk.offset;
bytes = node->VirtualChunk.bytes;
}
else
{
physical = node->Virtual.physical;
offset = 0;
bytes = node->Virtual.bytes;
}
gcmkERR_BREAK(gckOS_MemoryGetSGT(nodeObject->kernel->os, physical, offset, bytes, (gctPOINTER*)&sgt));
if (dma_map_sg(attachment->dev, sgt->sgl, sgt->nents, direction) == 0)
{
sg_free_table(sgt);
kfree(sgt);
sgt = gcvNULL;
gcmkERR_BREAK(gcvSTATUS_GENERIC_IO);
}
}
while (gcvFALSE);
return sgt;
}
static void _dmabuf_unmap(struct dma_buf_attachment *attachment,
struct sg_table *sgt,
enum dma_data_direction direction)
{
dma_unmap_sg(attachment->dev, sgt->sgl, sgt->nents, direction);
sg_free_table(sgt);
kfree(sgt);
}
static int _dmabuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
gckVIDMEM_NODE nodeObject = dmabuf->priv;
gcuVIDMEM_NODE_PTR node = nodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gctPHYS_ADDR physical = gcvNULL;
gctSIZE_T skipPages = vma->vm_pgoff;
gctSIZE_T numPages = PAGE_ALIGN(vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
gceSTATUS status = gcvSTATUS_OK;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
physical = node->VidMem.parent->physical;
skipPages += (node->VidMem.offset >> PAGE_SHIFT);
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
physical = vidMemBlock->physical;
skipPages += (node->VirtualChunk.offset >> PAGE_SHIFT);
}
else
{
physical = node->Virtual.physical;
}
gcmkONERROR(gckOS_MemoryMmap(nodeObject->kernel->os, physical, skipPages, numPages, vma));
OnError:
return gcmIS_ERROR(status) ? -EINVAL : 0;
}
static void _dmabuf_release(struct dma_buf *dmabuf)
{
gckVIDMEM_NODE nodeObject = dmabuf->priv;
gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(nodeObject->kernel, nodeObject));
}
#if LINUX_VERSION_CODE <= KERNEL_VERSION(5,5,7)
static void *_dmabuf_kmap(struct dma_buf *dmabuf, unsigned long offset)
{
gckVIDMEM_NODE nodeObject = dmabuf->priv;
gcuVIDMEM_NODE_PTR node = nodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gctINT8_PTR kvaddr = gcvNULL;
gctPHYS_ADDR physical = gcvNULL;
gctSIZE_T bytes = 0;
offset = (offset << PAGE_SHIFT);
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
physical = node->VidMem.parent->physical;
offset += node->VidMem.offset;
bytes = node->VidMem.bytes;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
physical = vidMemBlock->physical;
offset += node->VirtualChunk.offset;
bytes = node->VirtualChunk.bytes;
}
else
{
physical = node->Virtual.physical;
bytes = node->Virtual.bytes;
}
if (gcmIS_SUCCESS(gckOS_CreateKernelMapping(
nodeObject->kernel->os, physical, 0, bytes, (gctPOINTER*)&kvaddr)))
{
kvaddr += offset;
}
return (gctPOINTER)kvaddr;
}
static void _dmabuf_kunmap(struct dma_buf *dmabuf, unsigned long offset, void *ptr)
{
gckVIDMEM_NODE nodeObject = dmabuf->priv;
gcuVIDMEM_NODE_PTR node = nodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gctINT8_PTR kvaddr = (gctINT8_PTR)ptr - (offset << PAGE_SHIFT);
gctPHYS_ADDR physical = gcvNULL;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
physical = node->VidMem.parent->physical;
kvaddr -= node->VidMem.offset;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
physical = vidMemBlock->physical;
kvaddr -= node->VirtualChunk.offset;
}
else
{
physical = node->Virtual.physical;
}
gcmkVERIFY_OK(gckOS_DestroyKernelMapping(
nodeObject->kernel->os, physical, (gctPOINTER*)&kvaddr));
}
#endif
static gceSTATUS
_DmaBufCacheOperation(
IN gckVIDMEM_NODE NodeObject,
IN gceCACHEOPERATION Operation
)
{
gceSTATUS status = gcvSTATUS_OK;
gctPHYS_ADDR physHandle = gcvNULL;
gctPOINTER logical = gcvNULL;
gctSIZE_T offset = 0;
gctSIZE_T size = 0;
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gctUINT32 processID;
gckOS_GetProcessID(&processID);
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
/* Reserved pool can't be cacheable */
gcmkONERROR(gcvSTATUS_OK);
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
physHandle = vidMemBlock->physical;
offset = node->VirtualChunk.offset;
size = node->VirtualChunk.bytes;
logical = node->VirtualChunk.logical;
}
else
{
physHandle = node->Virtual.physical;
size = node->Virtual.bytes;
logical = node->Virtual.logical;
}
switch (Operation)
{
case gcvCACHE_CLEAN:
gcmkONERROR(gckOS_CacheClean(NodeObject->kernel->os, processID,
physHandle, offset, logical, size));
break;
case gcvCACHE_INVALIDATE:
gcmkONERROR(gckOS_CacheInvalidate(NodeObject->kernel->os, processID,
physHandle, offset, logical, size));
break;
case gcvCACHE_FLUSH:
gcmkONERROR(gckOS_CacheFlush(NodeObject->kernel->os, processID,
physHandle, offset, logical, size));
break;
default:
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
break;
}
OnError:
return status;
}
static int _dmabuf_begin_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction direction)
{
gceSTATUS status = gcvSTATUS_OK;
if (direction == DMA_BIDIRECTIONAL || direction == DMA_FROM_DEVICE) {
/* CPU read, invalidate caches. */
gcmkONERROR(_DmaBufCacheOperation(dmabuf->priv, gcvCACHE_INVALIDATE));
}
OnError:
return gcmIS_ERROR(status) ? -EINVAL : 0;
}
static int _dmabuf_end_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction direction)
{
gceSTATUS status = gcvSTATUS_OK;
if (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE) {
/* Flush caches on CPU write. */
if (direction == DMA_BIDIRECTIONAL) {
gcmkONERROR(_DmaBufCacheOperation(dmabuf->priv, gcvCACHE_FLUSH));
} else {
gcmkONERROR(_DmaBufCacheOperation(dmabuf->priv, gcvCACHE_CLEAN));
}
}
OnError:
return gcmIS_ERROR(status) ? -EINVAL : 0;
}
static struct dma_buf_ops _dmabuf_ops =
{
.map_dma_buf = _dmabuf_map,
.unmap_dma_buf = _dmabuf_unmap,
.mmap = _dmabuf_mmap,
.release = _dmabuf_release,
.begin_cpu_access = _dmabuf_begin_cpu_access,
.end_cpu_access = _dmabuf_end_cpu_access,
#if LINUX_VERSION_CODE > KERNEL_VERSION(5,5,7)
# elif LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0)
.map = _dmabuf_kmap,
.unmap = _dmabuf_kunmap,
# elif LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)
.map_atomic = _dmabuf_kmap,
.unmap_atomic = _dmabuf_kunmap,
.map = _dmabuf_kmap,
.unmap = _dmabuf_kunmap,
# else
.kmap_atomic = _dmabuf_kmap,
.kunmap_atomic = _dmabuf_kunmap,
.kmap = _dmabuf_kmap,
.kunmap = _dmabuf_kunmap,
# endif
};
#endif
gceSTATUS
gckVIDMEM_NODE_Export(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
IN gctINT32 Flags,
OUT gctPOINTER *DmaBuf,
OUT gctINT32 *FD
)
{
#if defined(CONFIG_DMA_SHARED_BUFFER)
gceSTATUS status = gcvSTATUS_OK;
struct dma_buf *dmabuf = gcvNULL;
gcmkHEADER_ARG("Kernel=%p NodeObject=0x%x", Kernel, NodeObject);
dmabuf = NodeObject->dmabuf;
if (!dmabuf)
{
gctSIZE_T bytes = 0;
gctPHYS_ADDR physical = gcvNULL;
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
physical = node->VidMem.parent->physical;
bytes = node->VidMem.bytes;
/* Align export size. when allocate memory from VIDMEM, the actual node size may not same with aligned size. */
bytes = bytes & ~(PAGE_SIZE - 1);
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
physical = vidMemBlock->physical;
bytes = node->VirtualChunk.bytes;
}
else
{
physical = node->Virtual.physical;
bytes = node->Virtual.bytes;
}
/* Donot really get SGT, just check if the allocator support GetSGT. */
gcmkONERROR(gckOS_MemoryGetSGT(Kernel->os, physical, 0, 0, NULL));
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0)
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
exp_info.ops = &_dmabuf_ops;
exp_info.size = bytes;
exp_info.flags = Flags;
exp_info.priv = NodeObject;
dmabuf = dma_buf_export(&exp_info);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0)
dmabuf = dma_buf_export(NodeObject, &_dmabuf_ops, bytes, Flags, NULL);
#else
dmabuf = dma_buf_export(NodeObject, &_dmabuf_ops, bytes, Flags);
#endif
}
if (IS_ERR(dmabuf))
{
gcmkONERROR(gcvSTATUS_GENERIC_IO);
}
/* Reference this gckVIDMEM_NODE object. */
gckVIDMEM_NODE_Reference(Kernel, NodeObject);
NodeObject->dmabuf = dmabuf;
}
if (DmaBuf)
{
*DmaBuf = NodeObject->dmabuf;
}
if (FD)
{
gctINT fd = dma_buf_fd(dmabuf, Flags);
if (fd < 0)
{
gcmkONERROR(gcvSTATUS_GENERIC_IO);
}
*FD = fd;
}
OnError:
gcmkFOOTER_ARG("*DmaBuf=%p *FD=0x%x", gcmOPT_POINTER(DmaBuf), gcmOPT_VALUE(FD));
return status;
#else
gcmkFATAL("The kernel did NOT support CONFIG_DMA_SHARED_BUFFER");
return gcvSTATUS_NOT_SUPPORTED;
#endif
}
gceSTATUS
gckVIDMEM_NODE_Name(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
OUT gctUINT32 * Name
)
{
gceSTATUS status;
gctUINT32 name = 0;
gctPOINTER database = Kernel->db->nameDatabase;
gctPOINTER mutex = Kernel->db->nameDatabaseMutex;
gctBOOL acquired = gcvFALSE;
gctBOOL referenced = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X NodeObject=%p", Kernel, NodeObject);
gcmkVERIFY_ARGUMENT(Name != gcvNULL);
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
gcmkONERROR(gckVIDMEM_NODE_Reference(Kernel, NodeObject));
referenced = gcvTRUE;
if (NodeObject->name == 0)
{
/* Name this node. */
gcmkONERROR(gckKERNEL_AllocateIntegerId(database, NodeObject, &name));
NodeObject->name = name;
}
else
{
name = NodeObject->name;
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, NodeObject));
*Name = name;
gcmkFOOTER_ARG("*Name=%d", *Name);
return gcvSTATUS_OK;
OnError:
if (referenced)
{
gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, NodeObject));
}
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_Import(
IN gckKERNEL Kernel,
IN gctUINT32 Name,
OUT gckVIDMEM_NODE * NodeObject
)
{
gceSTATUS status;
gckVIDMEM_NODE node = gcvNULL;
gctPOINTER database = Kernel->db->nameDatabase;
gctPOINTER mutex = Kernel->db->nameDatabaseMutex;
gctBOOL acquired = gcvFALSE;
gctBOOL referenced = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X Name=%d", Kernel, Name);
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE));
acquired = gcvTRUE;
/* Lookup in database to get the node. */
gcmkONERROR(gckKERNEL_QueryIntegerId(database, Name, (gctPOINTER *)&node));
/* Reference the node. */
gcmkONERROR(gckVIDMEM_NODE_Reference(Kernel, node));
referenced = gcvTRUE;
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
acquired = gcvFALSE;
*NodeObject = node;
gcmkFOOTER_ARG("*NodeObject=%p", node);
return gcvSTATUS_OK;
OnError:
if (referenced)
{
gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, node));
}
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex));
}
gcmkFOOTER();
return status;
}
typedef struct _gcsVIDMEM_NODE_FDPRIVATE
{
gcsFDPRIVATE base;
gckKERNEL kernel;
gckVIDMEM_NODE node;
}
gcsVIDMEM_NODE_FDPRIVATE;
static gctINT
_ReleaseFdPrivate(
gcsFDPRIVATE_PTR FdPrivate
)
{
/* Cast private info. */
gcsVIDMEM_NODE_FDPRIVATE * private = (gcsVIDMEM_NODE_FDPRIVATE *) FdPrivate;
gckVIDMEM_NODE_Dereference(private->kernel, private->node);
gckOS_Free(private->kernel->os, private);
return 0;
}
/*******************************************************************************
**
** gckVIDMEM_NODE_GetFd
**
** Attach a gckVIDMEM_NODE object to a native fd.
**
** OUTPUT:
**
** gctUINT32 * Fd
** Pointer to a variable receiving a native fd from os.
*/
gceSTATUS
gckVIDMEM_NODE_GetFd(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
OUT gctINT * Fd
)
{
gceSTATUS status;
gctBOOL referenced = gcvFALSE;
gcsVIDMEM_NODE_FDPRIVATE * fdPrivate = gcvNULL;
gcmkHEADER_ARG("Kernel=0x%X NodeObject=%d", Kernel, NodeObject);
/* Reference node object. */
gcmkVERIFY_OK(gckVIDMEM_NODE_Reference(Kernel, NodeObject));
referenced = gcvTRUE;
/* Allocated fd owns a reference. */
gcmkONERROR(gckOS_Allocate(
Kernel->os,
gcmSIZEOF(gcsVIDMEM_NODE_FDPRIVATE),
(gctPOINTER *)&fdPrivate
));
fdPrivate->base.release = _ReleaseFdPrivate;
fdPrivate->kernel = Kernel;
fdPrivate->node = NodeObject;
/* Allocated fd owns a reference. */
gcmkONERROR(gckOS_GetFd("vidmem", &fdPrivate->base, Fd));
gcmkFOOTER_ARG("*Fd=%d", *Fd);
return gcvSTATUS_OK;
OnError:
if (referenced)
{
gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, NodeObject));
}
if (fdPrivate)
{
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, fdPrivate));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_WrapUserMemory(
IN gckKERNEL Kernel,
IN gcsUSER_MEMORY_DESC_PTR Desc,
IN gceVIDMEM_TYPE Type,
OUT gckVIDMEM_NODE * NodeObject,
OUT gctUINT64 * Bytes
)
{
gceSTATUS status = gcvSTATUS_OK;
gckVIDMEM_NODE nodeObject = gcvNULL;
gctBOOL found = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%x", Kernel);
gcmkVERIFY_ARGUMENT(Desc != gcvNULL);
#if defined(CONFIG_DMA_SHARED_BUFFER)
if (Desc->flag & gcvALLOC_FLAG_DMABUF)
{
struct dma_buf *dmabuf;
int fd = (int)Desc->handle;
if (fd >= 0)
{
/* Import dma buf handle. */
dmabuf = dma_buf_get(fd);
if (IS_ERR(dmabuf))
return PTR_ERR(dmabuf);
Desc->handle = -1;
Desc->dmabuf = gcmPTR_TO_UINT64(dmabuf);
dma_buf_put(dmabuf);
}
else
{
if (!Desc->dmabuf)
{
gcmkPRINT("Wrap user memory: invalid dmabuf from user.\n");
gcmkFOOTER();
return gcvSTATUS_INVALID_ARGUMENT;
}
dmabuf = gcmUINT64_TO_PTR(Desc->dmabuf);
}
if (dmabuf->ops == &_dmabuf_ops)
{
gctBOOL referenced = gcvFALSE;
nodeObject = dmabuf->priv;
do
{
/* Reference the node. */
gcmkERR_BREAK(gckVIDMEM_NODE_Reference(Kernel, nodeObject));
referenced = gcvTRUE;
found = gcvTRUE;
*NodeObject = nodeObject;
*Bytes = (gctUINT64)dmabuf->size;
}
while (gcvFALSE);
if (gcmIS_ERROR(status) && referenced)
{
gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, nodeObject));
}
}
}
#endif
if (!found)
{
gckOS os = Kernel->os;
gcuVIDMEM_NODE_PTR node = gcvNULL;
do
{
gctSIZE_T pageCountCpu = 0;
gctSIZE_T pageSizeCpu = 0;
gctPHYS_ADDR_T physicalAddress = 0;
gcmkVERIFY_OK(gckOS_GetPageSize(os, &pageSizeCpu));
/* Allocate an gcuVIDMEM_NODE union. */
gcmkERR_BREAK(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), (gctPOINTER*)&node));
gckOS_ZeroMemory(node, gcmSIZEOF(gcuVIDMEM_NODE));
/* Initialize gcuVIDMEM_NODE union for virtual memory. */
node->Virtual.kernel = Kernel;
/* Wrap Memory. */
gcmkERR_BREAK(
gckOS_WrapMemory(os,
Desc,
&node->Virtual.bytes,
&node->Virtual.physical,
&node->Virtual.contiguous,
&pageCountCpu));
/* Get base physical address. */
gcmkERR_BREAK(
gckOS_GetPhysicalFromHandle(os,
node->Virtual.physical,
0,
&physicalAddress));
/* Allocate handle for this video memory. */
gcmkERR_BREAK(gckVIDMEM_NODE_Construct(
Kernel,
node,
Type,
gcvPOOL_VIRTUAL,
&nodeObject
));
node->Virtual.pageCount = (pageCountCpu * pageSizeCpu -
(physicalAddress & (pageSizeCpu - 1) & ~(4096 - 1))) >> 12;
*NodeObject = nodeObject;
*Bytes = (gctUINT64)node->Virtual.bytes;
}
while (gcvFALSE);
if (gcmIS_ERROR(status) && node)
{
/* Free the structure. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node));
}
}
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckVIDMEM_NODE_SetCommitStamp(
IN gckKERNEL Kernel,
IN gceENGINE Engine,
IN gckVIDMEM_NODE NodeObject,
IN gctUINT64 CommitStamp
)
{
NodeObject->sync[Engine].commitStamp = CommitStamp;
return gcvSTATUS_OK;
}
gceSTATUS
gckVIDMEM_NODE_GetCommitStamp(
IN gckKERNEL Kernel,
IN gceENGINE Engine,
IN gckVIDMEM_NODE NodeObject,
OUT gctUINT64_PTR CommitStamp
)
{
*CommitStamp = NodeObject->sync[Engine].commitStamp;
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckVIDMEM_NODE_Find
**
** Find gckVIDMEM_NODE object according to GPU address of specified core.
**
** INPUT:
**
** gckKERNEL Kernel
** Kernel object, specifies core.
**
** gctUINT32 Address
** GPU address to search.
**
** OUTPUT:
**
** gckVIDMEM_NODE * NodeObject
** Pointer to a variable hold found video memory node.
**
** gctUINT32 * Offset
** The offset of specified GPU address in found video memory node.
*/
gceSTATUS
gckVIDMEM_NODE_Find(
IN gckKERNEL Kernel,
IN gctUINT32 Address,
OUT gckVIDMEM_NODE * NodeObject,
OUT gctUINT32 * Offset
)
{
gceSTATUS status = gcvSTATUS_NOT_FOUND;
gckVIDMEM_NODE nodeObject = gcvNULL;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gckVIDMEM_BLOCK vidMemBlock = gcvNULL;
gcsLISTHEAD_PTR pos;
gceHARDWARE_TYPE hwType;
gcmkVERIFY_OK(
gckKERNEL_GetHardwareType(Kernel,
&hwType));
gcmkVERIFY_OK(
gckOS_AcquireMutex(Kernel->os,
Kernel->db->videoMemListMutex,
gcvINFINITE));
gcmkLIST_FOR_EACH(pos, &Kernel->db->videoMemList)
{
nodeObject = (gckVIDMEM_NODE)gcmCONTAINEROF(pos, struct _gcsVIDMEM_NODE, link);
node = nodeObject->node;
vidMemBlock = node->VirtualChunk.parent;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
if (!node->VidMem.locked)
{
/* Don't check against unlocked node. */
continue;
}
if (Address >= node->VidMem.address &&
Address <= node->VidMem.address + node->VidMem.bytes - 1)
{
*NodeObject = nodeObject;
if (Offset)
{
*Offset = Address - node->VidMem.address;
}
status = gcvSTATUS_OK;
break;
}
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
if (!node->VirtualChunk.lockeds[hwType])
{
/* Don't check against unlocked node. */
continue;
}
if (Address >= node->VirtualChunk.addresses[hwType] &&
Address <= node->VirtualChunk.addresses[hwType] + node->VirtualChunk.bytes - 1)
{
*NodeObject = nodeObject;
if (Offset)
{
*Offset = Address - node->VirtualChunk.addresses[hwType];
}
status = gcvSTATUS_OK;
break;
}
}
else
{
if (!node->Virtual.lockeds[hwType])
{
/* Don't check against unlocked node. */
continue;
}
if (Address >= node->Virtual.addresses[hwType] &&
(Address <= node->Virtual.addresses[hwType] + node->Virtual.bytes - 1))
{
*NodeObject = nodeObject;
if (Offset)
{
*Offset = Address - node->Virtual.addresses[hwType];
}
status = gcvSTATUS_OK;
break;
}
}
}
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex));
return status;
}
gceSTATUS
gckVIDMEM_NODE_IsContiguous(
IN gckKERNEL Kernel,
IN gckVIDMEM_NODE NodeObject,
OUT gctBOOL * Contiguous
)
{
gceSTATUS status;
gckOS os = Kernel->os;
gctBOOL acquired = gcvFALSE;
gcuVIDMEM_NODE_PTR node = NodeObject->node;
gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent;
gcmkHEADER();
/* Grab the mutex. */
gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE));
acquired = gcvTRUE;
*Contiguous = gcvFALSE;
if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM)
{
*Contiguous = gcvTRUE;
}
else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK)
{
if (vidMemBlock->contiguous)
{
*Contiguous = gcvTRUE;
}
}
else
{
if (node->Virtual.contiguous)
{
*Contiguous = gcvTRUE;
}
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
gcmkFOOTER();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}