| /**************************************************************************** |
| * |
| * The MIT License (MIT) |
| * |
| * Copyright (c) 2014 - 2020 Vivante Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in |
| * all copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| * DEALINGS IN THE SOFTWARE. |
| * |
| ***************************************************************************** |
| * |
| * The GPL License (GPL) |
| * |
| * Copyright (C) 2014 - 2020 Vivante Corporation |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; either version 2 |
| * of the License, or (at your option) any later version. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| * |
| ***************************************************************************** |
| * |
| * Note: This software is released under dual MIT and GPL licenses. A |
| * recipient may use this file under the terms of either the MIT license or |
| * GPL License. If you wish to use only one license not the other, you can |
| * indicate your decision by deleting one of the above license notices in your |
| * version of this file. |
| * |
| *****************************************************************************/ |
| |
| |
| #include "gc_hal_kernel_precomp.h" |
| |
| #define _GC_OBJ_ZONE gcvZONE_VIDMEM |
| |
| /******************************************************************************\ |
| ******************************* Private Functions ****************************** |
| \******************************************************************************/ |
| |
| /******************************************************************************* |
| ** |
| ** _Split |
| ** |
| ** Split a node on the required byte boundary. |
| ** |
| ** INPUT: |
| ** |
| ** gckOS Os |
| ** Pointer to an gckOS object. |
| ** |
| ** gcuVIDMEM_NODE_PTR Node |
| ** Pointer to the node to split. |
| ** |
| ** gctSIZE_T Bytes |
| ** Number of bytes to keep in the node. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| ** |
| ** RETURNS: |
| ** |
| ** gctBOOL |
| ** gcvTRUE if the node was split successfully, or gcvFALSE if there is an |
| ** error. |
| ** |
| */ |
| static gctBOOL |
| _Split( |
| IN gckOS Os, |
| IN gcuVIDMEM_NODE_PTR Node, |
| IN gctSIZE_T Bytes |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node; |
| gctPOINTER pointer = gcvNULL; |
| |
| /* Make sure the byte boundary makes sense. */ |
| if ((Bytes <= 0) || (Bytes > Node->VidMem.bytes)) |
| { |
| return gcvFALSE; |
| } |
| |
| /* Allocate a new gcuVIDMEM_NODE object. */ |
| if (gcmIS_ERROR(gckOS_Allocate(Os, |
| gcmSIZEOF(gcuVIDMEM_NODE), |
| &pointer))) |
| { |
| /* Error. */ |
| return gcvFALSE; |
| } |
| |
| node = pointer; |
| |
| /* Initialize gcuVIDMEM_NODE structure. */ |
| node->VidMem.offset = Node->VidMem.offset + Bytes; |
| node->VidMem.bytes = Node->VidMem.bytes - Bytes; |
| node->VidMem.alignment = 0; |
| node->VidMem.locked = 0; |
| node->VidMem.parent = Node->VidMem.parent; |
| node->VidMem.pool = Node->VidMem.pool; |
| node->VidMem.processID = 0; |
| node->VidMem.logical = gcvNULL; |
| node->VidMem.kvaddr = gcvNULL; |
| |
| /* Insert node behind specified node. */ |
| node->VidMem.next = Node->VidMem.next; |
| node->VidMem.prev = Node; |
| Node->VidMem.next = node->VidMem.next->VidMem.prev = node; |
| |
| /* Insert free node behind specified node. */ |
| node->VidMem.nextFree = Node->VidMem.nextFree; |
| node->VidMem.prevFree = Node; |
| Node->VidMem.nextFree = node->VidMem.nextFree->VidMem.prevFree = node; |
| |
| /* Adjust size of specified node. */ |
| Node->VidMem.bytes = Bytes; |
| |
| /* Success. */ |
| return gcvTRUE; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** _Merge |
| ** |
| ** Merge two adjacent nodes together. |
| ** |
| ** INPUT: |
| ** |
| ** gckOS Os |
| ** Pointer to an gckOS object. |
| ** |
| ** gcuVIDMEM_NODE_PTR Node |
| ** Pointer to the first of the two nodes to merge. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| ** |
| */ |
| static gceSTATUS |
| _Merge( |
| IN gckOS Os, |
| IN gcuVIDMEM_NODE_PTR Node |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node; |
| gceSTATUS status; |
| |
| /* Save pointer to next node. */ |
| node = Node->VidMem.next; |
| |
| /* This is a good time to make sure the heap is not corrupted. */ |
| if (Node->VidMem.offset + Node->VidMem.bytes != node->VidMem.offset) |
| { |
| /* Corrupted heap. */ |
| gcmkASSERT( |
| Node->VidMem.offset + Node->VidMem.bytes == node->VidMem.offset); |
| return gcvSTATUS_HEAP_CORRUPTED; |
| } |
| |
| /* Adjust byte count. */ |
| Node->VidMem.bytes += node->VidMem.bytes; |
| |
| /* Unlink next node from linked list. */ |
| Node->VidMem.next = node->VidMem.next; |
| Node->VidMem.nextFree = node->VidMem.nextFree; |
| |
| Node->VidMem.next->VidMem.prev = |
| Node->VidMem.nextFree->VidMem.prevFree = Node; |
| |
| /* Free next node. */ |
| status = gcmkOS_SAFE_FREE(Os, node); |
| return status; |
| } |
| |
| /******************************************************************************\ |
| ******************************* gckVIDMEM API Code ****************************** |
| \******************************************************************************/ |
| |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_Construct |
| ** |
| ** Construct a new gckVIDMEM object. |
| ** |
| ** INPUT: |
| ** |
| ** gckOS Os |
| ** Pointer to an gckOS object. |
| ** |
| ** gctPHYS_ADDR_T PhysicalBase |
| ** Base physical address for the video memory heap. |
| ** |
| ** gctSIZE_T Bytes |
| ** Number of bytes in the video memory heap. |
| ** |
| ** gctSIZE_T Threshold |
| ** Minimum number of bytes beyond am allocation before the node is |
| ** split. Can be used as a minimum alignment requirement. |
| ** |
| ** gctSIZE_T BankSize |
| ** Number of bytes per physical memory bank. Used by bank |
| ** optimization. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckVIDMEM * Memory |
| ** Pointer to a variable that will hold the pointer to the gckVIDMEM |
| ** object. |
| */ |
| gceSTATUS |
| gckVIDMEM_Construct( |
| IN gckOS Os, |
| IN gctPHYS_ADDR_T PhysicalBase, |
| IN gctSIZE_T Bytes, |
| IN gctSIZE_T Threshold, |
| IN gctSIZE_T BankSize, |
| OUT gckVIDMEM * Memory |
| ) |
| { |
| gckVIDMEM memory = gcvNULL; |
| gceSTATUS status; |
| gcuVIDMEM_NODE_PTR node; |
| gctINT i, banks = 0; |
| gctPOINTER pointer = gcvNULL; |
| gctUINT32 heapBytes; |
| gctUINT32 bankSize; |
| gctUINT32 base = 0; |
| |
| gcmkHEADER_ARG("Os=0x%x PhysicalBase=%12llx Bytes=%lu Threshold=%lu " |
| "BankSize=%lu", |
| Os, PhysicalBase, Bytes, Threshold, BankSize); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Os, gcvOBJ_OS); |
| gcmkVERIFY_ARGUMENT(Bytes > 0); |
| gcmkVERIFY_ARGUMENT(Memory != gcvNULL); |
| |
| gcmkSAFECASTSIZET(heapBytes, Bytes); |
| gcmkSAFECASTSIZET(bankSize, BankSize); |
| |
| /* Allocate the gckVIDMEM object. */ |
| gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(struct _gckVIDMEM), &pointer)); |
| gckOS_ZeroMemory(pointer, gcmSIZEOF(struct _gckVIDMEM)); |
| |
| memory = pointer; |
| |
| /* Initialize the gckVIDMEM object. */ |
| memory->object.type = gcvOBJ_VIDMEM; |
| memory->os = Os; |
| |
| /* Set video memory heap information. */ |
| memory->physicalBase = PhysicalBase; |
| memory->bytes = heapBytes; |
| memory->freeBytes = heapBytes; |
| memory->minFreeBytes = heapBytes; |
| memory->capability = ~0u; |
| memory->threshold = Threshold; |
| memory->mutex = gcvNULL; |
| |
| /* Walk all possible banks. */ |
| for (i = 0; i < gcmCOUNTOF(memory->sentinel); ++i) |
| { |
| gctUINT32 bytes; |
| |
| if (BankSize == 0) |
| { |
| /* Use all bytes for the first bank. */ |
| bytes = heapBytes; |
| } |
| else |
| { |
| /* Compute number of bytes for this bank. */ |
| bytes = gcmALIGN(base + 1, bankSize) - base; |
| |
| if (bytes > heapBytes) |
| { |
| /* Make sure we don't exceed the total number of bytes. */ |
| bytes = heapBytes; |
| } |
| } |
| |
| if (bytes == 0) |
| { |
| /* Mark heap is not used. */ |
| memory->sentinel[i].VidMem.next = |
| memory->sentinel[i].VidMem.prev = |
| memory->sentinel[i].VidMem.nextFree = |
| memory->sentinel[i].VidMem.prevFree = gcvNULL; |
| continue; |
| } |
| |
| /* Allocate one gcuVIDMEM_NODE union. */ |
| gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer)); |
| |
| node = pointer; |
| |
| /* Initialize gcuVIDMEM_NODE union. */ |
| node->VidMem.parent = memory; |
| |
| node->VidMem.next = |
| node->VidMem.prev = |
| node->VidMem.nextFree = |
| node->VidMem.prevFree = &memory->sentinel[i]; |
| |
| node->VidMem.offset = base; |
| node->VidMem.bytes = bytes; |
| node->VidMem.alignment = 0; |
| node->VidMem.pool = gcvPOOL_UNKNOWN; |
| |
| node->VidMem.locked = 0; |
| |
| node->VidMem.processID = 0; |
| node->VidMem.logical = gcvNULL; |
| |
| node->VidMem.kvaddr = gcvNULL; |
| |
| /* Initialize the linked list of nodes. */ |
| memory->sentinel[i].VidMem.next = |
| memory->sentinel[i].VidMem.prev = |
| memory->sentinel[i].VidMem.nextFree = |
| memory->sentinel[i].VidMem.prevFree = node; |
| |
| /* Mark sentinel. */ |
| memory->sentinel[i].VidMem.bytes = 0; |
| |
| /* Adjust address for next bank. */ |
| base += bytes; |
| heapBytes -= bytes; |
| banks ++; |
| } |
| |
| /* Assign all the bank mappings. */ |
| memory->mapping[gcvVIDMEM_TYPE_COLOR_BUFFER] = banks - 1; |
| memory->mapping[gcvVIDMEM_TYPE_BITMAP] = banks - 1; |
| |
| if (banks > 1) --banks; |
| memory->mapping[gcvVIDMEM_TYPE_DEPTH_BUFFER] = banks - 1; |
| memory->mapping[gcvVIDMEM_TYPE_HZ_BUFFER] = banks - 1; |
| |
| if (banks > 1) --banks; |
| memory->mapping[gcvVIDMEM_TYPE_TEXTURE] = banks - 1; |
| |
| if (banks > 1) --banks; |
| memory->mapping[gcvVIDMEM_TYPE_VERTEX_BUFFER] = banks - 1; |
| |
| if (banks > 1) --banks; |
| memory->mapping[gcvVIDMEM_TYPE_INDEX_BUFFER] = banks - 1; |
| |
| if (banks > 1) --banks; |
| memory->mapping[gcvVIDMEM_TYPE_TILE_STATUS] = banks - 1; |
| |
| if (banks > 1) --banks; |
| memory->mapping[gcvVIDMEM_TYPE_COMMAND] = banks - 1; |
| |
| if (banks > 1) --banks; |
| memory->mapping[gcvVIDMEM_TYPE_GENERIC] = 0; |
| |
| memory->mapping[gcvVIDMEM_TYPE_ICACHE] = 0; |
| memory->mapping[gcvVIDMEM_TYPE_TXDESC] = 0; |
| memory->mapping[gcvVIDMEM_TYPE_FENCE] = 0; |
| memory->mapping[gcvVIDMEM_TYPE_TFBHEADER] = 0; |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "[GALCORE] INDEX: bank %d", |
| memory->mapping[gcvVIDMEM_TYPE_INDEX_BUFFER]); |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "[GALCORE] VERTEX: bank %d", |
| memory->mapping[gcvVIDMEM_TYPE_VERTEX_BUFFER]); |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "[GALCORE] TEXTURE: bank %d", |
| memory->mapping[gcvVIDMEM_TYPE_TEXTURE]); |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "[GALCORE] RENDER_TARGET: bank %d", |
| memory->mapping[gcvVIDMEM_TYPE_COLOR_BUFFER]); |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "[GALCORE] DEPTH: bank %d", |
| memory->mapping[gcvVIDMEM_TYPE_DEPTH_BUFFER]); |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "[GALCORE] TILE_STATUS: bank %d", |
| memory->mapping[gcvVIDMEM_TYPE_TILE_STATUS]); |
| |
| /* Allocate the mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(Os, &memory->mutex)); |
| |
| /* Return pointer to the gckVIDMEM object. */ |
| *Memory = memory; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Memory=0x%x", *Memory); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Roll back. */ |
| if (memory != gcvNULL) |
| { |
| if (memory->mutex != gcvNULL) |
| { |
| /* Delete the mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Os, memory->mutex)); |
| } |
| |
| for (i = 0; i < banks; ++i) |
| { |
| /* Free the heap. */ |
| gcmkASSERT(memory->sentinel[i].VidMem.next != gcvNULL); |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory->sentinel[i].VidMem.next)); |
| } |
| |
| /* Free the object. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_Destroy |
| ** |
| ** Destroy an gckVIDMEM object. |
| ** |
| ** INPUT: |
| ** |
| ** gckVIDMEM Memory |
| ** Pointer to an gckVIDMEM object to destroy. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckVIDMEM_Destroy( |
| IN gckVIDMEM Memory |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node, next; |
| gctINT i; |
| |
| gcmkHEADER_ARG("Memory=0x%x", Memory); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM); |
| |
| /* Walk all sentinels. */ |
| for (i = 0; i < gcmCOUNTOF(Memory->sentinel); ++i) |
| { |
| /* Bail out of the heap is not used. */ |
| if (Memory->sentinel[i].VidMem.next == gcvNULL) |
| { |
| break; |
| } |
| |
| /* Walk all the nodes until we reach the sentinel. */ |
| for (node = Memory->sentinel[i].VidMem.next; |
| node->VidMem.bytes != 0; |
| node = next) |
| { |
| /* Save pointer to the next node. */ |
| next = node->VidMem.next; |
| |
| /* Free the node. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, node)); |
| } |
| } |
| |
| /* Free the mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Memory->os, Memory->mutex)); |
| |
| /* Mark the object as unknown. */ |
| Memory->object.type = gcvOBJ_UNKNOWN; |
| |
| /* Free the gckVIDMEM object. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, Memory)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| #if gcdENABLE_BANK_ALIGNMENT |
| |
| #if !gcdBANK_BIT_START |
| #error gcdBANK_BIT_START not defined. |
| #endif |
| |
| #if !gcdBANK_BIT_END |
| #error gcdBANK_BIT_END not defined. |
| #endif |
| /******************************************************************************* |
| ** _GetSurfaceBankAlignment |
| ** |
| ** Return the required offset alignment required to the make BaseAddress |
| ** aligned properly. |
| ** |
| ** INPUT: |
| ** |
| ** gckOS Os |
| ** Pointer to gcoOS object. |
| ** |
| ** gceVIDMEM_TYPE Type |
| ** Type of allocation. |
| ** |
| ** gctUINT32 BaseAddress |
| ** Base address of current video memory node. |
| ** |
| ** OUTPUT: |
| ** |
| ** gctUINT32_PTR AlignmentOffset |
| ** Pointer to a variable that will hold the number of bytes to skip in |
| ** the current video memory node in order to make the alignment bank |
| ** aligned. |
| */ |
| static gceSTATUS |
| _GetSurfaceBankAlignment( |
| IN gckKERNEL Kernel, |
| IN gceVIDMEM_TYPE Type, |
| IN gctUINT32 BaseAddress, |
| OUT gctUINT32_PTR AlignmentOffset |
| ) |
| { |
| gctUINT32 bank; |
| /* To retrieve the bank. */ |
| static const gctUINT32 bankMask = (0xFFFFFFFF << gcdBANK_BIT_START) |
| ^ (0xFFFFFFFF << (gcdBANK_BIT_END + 1)); |
| |
| /* To retrieve the bank and all the lower bytes. */ |
| static const gctUINT32 byteMask = ~(0xFFFFFFFF << (gcdBANK_BIT_END + 1)); |
| |
| gcmkHEADER_ARG("Type=%d BaseAddress=0x%x ", Type, BaseAddress); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_ARGUMENT(AlignmentOffset != gcvNULL); |
| |
| switch (Type) |
| { |
| case gcvVIDMEM_TYPE_COLOR_BUFFER: |
| bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START); |
| |
| /* Align to the first bank. */ |
| *AlignmentOffset = (bank == 0) ? |
| 0 : |
| ((1 << (gcdBANK_BIT_END + 1)) + 0) - (BaseAddress & byteMask); |
| break; |
| |
| case gcvVIDMEM_TYPE_DEPTH_BUFFER: |
| bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START); |
| |
| /* Align to the third bank. */ |
| *AlignmentOffset = (bank == 2) ? |
| 0 : |
| ((1 << (gcdBANK_BIT_END + 1)) + (2 << gcdBANK_BIT_START)) - (BaseAddress & byteMask); |
| |
| /* Minimum 256 byte alignment needed for fast_msaa. */ |
| if ((gcdBANK_CHANNEL_BIT > 7) || |
| ((gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_FAST_MSAA) != gcvSTATUS_TRUE) && |
| (gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_SMALL_MSAA) != gcvSTATUS_TRUE))) |
| { |
| /* Add a channel offset at the channel bit. */ |
| *AlignmentOffset += (1 << gcdBANK_CHANNEL_BIT); |
| } |
| break; |
| |
| default: |
| /* no alignment needed. */ |
| *AlignmentOffset = 0; |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER_ARG("*AlignmentOffset=%u", *AlignmentOffset); |
| return gcvSTATUS_OK; |
| } |
| #endif |
| |
| static gcuVIDMEM_NODE_PTR |
| _FindNode( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM Memory, |
| IN gctINT Bank, |
| IN gctSIZE_T Bytes, |
| IN gceVIDMEM_TYPE Type, |
| IN OUT gctUINT32_PTR Alignment |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node; |
| gctUINT32 alignment; |
| |
| #if gcdENABLE_BANK_ALIGNMENT |
| gctUINT32 bankAlignment; |
| gceSTATUS status; |
| #endif |
| |
| if (Memory->sentinel[Bank].VidMem.nextFree == gcvNULL) |
| { |
| /* No free nodes left. */ |
| return gcvNULL; |
| } |
| |
| #if gcdENABLE_BANK_ALIGNMENT |
| /* Walk all free nodes until we have one that is big enough or we have |
| ** reached the sentinel. */ |
| for (node = Memory->sentinel[Bank].VidMem.nextFree; |
| node->VidMem.bytes != 0; |
| node = node->VidMem.nextFree) |
| { |
| if (node->VidMem.bytes < Bytes) |
| { |
| continue; |
| } |
| |
| gcmkONERROR(_GetSurfaceBankAlignment( |
| Kernel, |
| Type, |
| (gctUINT32)(node->VidMem.parent->physicalBase + node->VidMem.offset), |
| &bankAlignment)); |
| |
| bankAlignment = gcmALIGN(bankAlignment, *Alignment); |
| |
| /* Compute number of bytes to skip for alignment. */ |
| alignment = (*Alignment == 0) |
| ? 0 |
| : (*Alignment - (node->VidMem.offset % *Alignment)); |
| |
| if (alignment == *Alignment) |
| { |
| /* Node is already aligned. */ |
| alignment = 0; |
| } |
| |
| if (node->VidMem.bytes >= Bytes + alignment + bankAlignment) |
| { |
| /* This node is big enough. */ |
| *Alignment = alignment + bankAlignment; |
| return node; |
| } |
| } |
| #endif |
| |
| /* Walk all free nodes until we have one that is big enough or we have |
| reached the sentinel. */ |
| for (node = Memory->sentinel[Bank].VidMem.nextFree; |
| node->VidMem.bytes != 0; |
| node = node->VidMem.nextFree) |
| { |
| gctUINT offset; |
| |
| gctINT modulo; |
| |
| gcmkSAFECASTSIZET(offset, node->VidMem.offset); |
| |
| modulo = gckMATH_ModuloInt(offset, *Alignment); |
| |
| /* Compute number of bytes to skip for alignment. */ |
| alignment = (*Alignment == 0) ? 0 : (*Alignment - modulo); |
| |
| if (alignment == *Alignment) |
| { |
| /* Node is already aligned. */ |
| alignment = 0; |
| } |
| |
| if (node->VidMem.bytes >= Bytes + alignment) |
| { |
| /* This node is big enough. */ |
| *Alignment = alignment; |
| return node; |
| } |
| } |
| |
| #if gcdENABLE_BANK_ALIGNMENT |
| OnError: |
| #endif |
| /* Not enough memory. */ |
| return gcvNULL; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_AllocateLinear |
| ** |
| ** Allocate linear memory from the gckVIDMEM object. |
| ** |
| ** INPUT: |
| ** |
| ** gckVIDMEM Memory |
| ** Pointer to an gckVIDMEM object. |
| ** |
| ** gctSIZE_T Bytes |
| ** Number of bytes to allocate. |
| ** |
| ** gctUINT32 Alignment |
| ** Byte alignment for allocation. |
| ** |
| ** gceVIDMEM_TYPE Type |
| ** Type of surface to allocate (use by bank optimization). |
| ** |
| ** gctUINT32 Flag |
| ** Flag of allocatetion. |
| ** |
| ** gctBOOL Specified |
| ** If user must use this pool, it should set Specified to gcvTRUE, |
| ** otherwise allocator may reserve some memory for other usage, such |
| ** as small block size allocation request. |
| ** |
| ** OUTPUT: |
| ** |
| ** gcuVIDMEM_NODE_PTR * Node |
| ** Pointer to a variable that will hold the allocated memory node. |
| */ |
| static gceSTATUS |
| gckVIDMEM_AllocateLinear( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM Memory, |
| IN gctSIZE_T Bytes, |
| IN gctUINT32 Alignment, |
| IN gceVIDMEM_TYPE Type, |
| IN gctUINT32 Flag, |
| IN gctBOOL Specified, |
| OUT gcuVIDMEM_NODE_PTR * Node |
| ) |
| { |
| gceSTATUS status; |
| gcuVIDMEM_NODE_PTR node; |
| gctUINT32 alignment; |
| gctINT bank, i; |
| gctBOOL acquired = gcvFALSE; |
| gctUINT64 mappingInOne = 1; |
| |
| gcmkHEADER_ARG("Memory=0x%x Bytes=%lu Alignment=%u Type=%d", |
| Memory, Bytes, Alignment, Type); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM); |
| gcmkVERIFY_ARGUMENT(Bytes > 0); |
| gcmkVERIFY_ARGUMENT(Node != gcvNULL); |
| gcmkVERIFY_ARGUMENT(Type < gcvVIDMEM_TYPE_COUNT); |
| |
| /* Acquire the mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(Memory->os, Memory->mutex, gcvINFINITE)); |
| |
| acquired = gcvTRUE; |
| |
| if (Bytes > Memory->freeBytes) |
| { |
| /* Not enough memory. */ |
| status = gcvSTATUS_OUT_OF_MEMORY; |
| goto OnError; |
| } |
| |
| #if gcdSMALL_BLOCK_SIZE |
| if ((Memory->freeBytes < (Memory->bytes/gcdRATIO_FOR_SMALL_MEMORY)) |
| && (Bytes >= gcdSMALL_BLOCK_SIZE) |
| && (Specified == gcvFALSE) |
| ) |
| { |
| /* The left memory is for small memory.*/ |
| status = gcvSTATUS_OUT_OF_MEMORY; |
| goto OnError; |
| } |
| #endif |
| |
| /* Find the default bank for this surface type. */ |
| gcmkASSERT((gctINT) Type < gcmCOUNTOF(Memory->mapping)); |
| bank = Memory->mapping[Type]; |
| alignment = Alignment; |
| |
| /* Find a free node in the default bank. */ |
| node = _FindNode(Kernel, Memory, bank, Bytes, Type, &alignment); |
| |
| /* Out of memory? */ |
| if (node == gcvNULL) |
| { |
| /* Walk all lower banks. */ |
| for (i = bank - 1; i >= 0; --i) |
| { |
| /* Find a free node inside the current bank. */ |
| node = _FindNode(Kernel, Memory, i, Bytes, Type, &alignment); |
| if (node != gcvNULL) |
| { |
| break; |
| } |
| } |
| } |
| |
| if (node == gcvNULL) |
| { |
| /* Walk all upper banks. */ |
| for (i = bank + 1; i < gcmCOUNTOF(Memory->sentinel); ++i) |
| { |
| if (Memory->sentinel[i].VidMem.nextFree == gcvNULL) |
| { |
| /* Abort when we reach unused banks. */ |
| break; |
| } |
| |
| /* Find a free node inside the current bank. */ |
| node = _FindNode(Kernel, Memory, i, Bytes, Type, &alignment); |
| if (node != gcvNULL) |
| { |
| break; |
| } |
| } |
| } |
| |
| if (node == gcvNULL) |
| { |
| /* Out of memory. */ |
| status = gcvSTATUS_OUT_OF_MEMORY; |
| goto OnError; |
| } |
| |
| /* Do we have an alignment? */ |
| if (alignment > 0) |
| { |
| /* Split the node so it is aligned. */ |
| if (_Split(Memory->os, node, alignment)) |
| { |
| /* Successful split, move to aligned node. */ |
| node = node->VidMem.next; |
| |
| /* Remove alignment. */ |
| alignment = 0; |
| } |
| } |
| |
| /* Do we have enough memory after the allocation to split it? */ |
| if (node->VidMem.bytes - Bytes > Memory->threshold) |
| { |
| /* Adjust the node size. */ |
| _Split(Memory->os, node, Bytes); |
| } |
| |
| /* Remove the node from the free list. */ |
| node->VidMem.prevFree->VidMem.nextFree = node->VidMem.nextFree; |
| node->VidMem.nextFree->VidMem.prevFree = node->VidMem.prevFree; |
| node->VidMem.nextFree = |
| node->VidMem.prevFree = gcvNULL; |
| |
| /* Fill in the information. */ |
| node->VidMem.alignment = alignment; |
| node->VidMem.parent = Memory; |
| node->VidMem.logical = gcvNULL; |
| gcmkONERROR(gckOS_GetProcessID(&node->VidMem.processID)); |
| |
| /* Adjust the number of free bytes. */ |
| Memory->freeBytes -= node->VidMem.bytes; |
| |
| if (Memory->freeBytes < Memory->minFreeBytes) |
| { |
| Memory->minFreeBytes = Memory->freeBytes; |
| } |
| |
| gckOS_QueryOption(Memory->os, "allMapInOne", &mappingInOne); |
| if (!mappingInOne) |
| { |
| gcmkONERROR(gckOS_RequestReservedMemory( |
| Memory->os, |
| Memory->physicalBase + node->VidMem.offset, |
| node->VidMem.bytes, |
| "gal reserved memory", |
| gcvTRUE, |
| &node->VidMem.physical |
| )); |
| } |
| |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex)); |
| |
| /* Return the pointer to the node. */ |
| *Node = node; |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Allocated %u bytes @ 0x%x [0x%08X]", |
| node->VidMem.bytes, node, node->VidMem.offset); |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Node=0x%x", *Node); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_AllocateVirtual |
| ** |
| ** Construct a new gcuVIDMEM_NODE union for virtual memory. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** gctSIZE_T Bytes |
| ** Number of byte to allocate. |
| ** |
| ** OUTPUT: |
| ** |
| ** gcuVIDMEM_NODE_PTR * Node |
| ** Pointer to a variable that receives the gcuVIDMEM_NODE union pointer. |
| */ |
| static gceSTATUS |
| gckVIDMEM_AllocateVirtual( |
| IN gckKERNEL Kernel, |
| IN gctUINT32 Flag, |
| IN gctSIZE_T Bytes, |
| OUT gcuVIDMEM_NODE_PTR * Node |
| ) |
| { |
| gckOS os; |
| gceSTATUS status; |
| gcuVIDMEM_NODE_PTR node = gcvNULL; |
| gctPOINTER pointer = gcvNULL; |
| gctINT i; |
| |
| gcmkHEADER_ARG("Kernel=0x%x Flag=%x Bytes=%lu", Kernel, Flag, Bytes); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); |
| gcmkVERIFY_ARGUMENT(Bytes > 0); |
| gcmkVERIFY_ARGUMENT(Node != gcvNULL); |
| |
| /* Extract the gckOS object pointer. */ |
| os = Kernel->os; |
| gcmkVERIFY_OBJECT(os, gcvOBJ_OS); |
| |
| /* Allocate an gcuVIDMEM_NODE union. */ |
| gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer)); |
| |
| node = pointer; |
| |
| /* Initialize gcuVIDMEM_NODE union for virtual memory. */ |
| node->Virtual.kernel = Kernel; |
| node->Virtual.contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS; |
| node->Virtual.logical = gcvNULL; |
| node->Virtual.kvaddr = gcvNULL; |
| node->Virtual.bytes = Bytes; |
| node->Virtual.secure = (Flag & gcvALLOC_FLAG_SECURITY) != 0; |
| node->Virtual.onFault = (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT) != 0; |
| |
| for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) |
| { |
| node->Virtual.lockeds[i] = 0; |
| node->Virtual.pageTables[i] = gcvNULL; |
| } |
| |
| /* Allocate the virtual memory. */ |
| gcmkONERROR( |
| gckOS_AllocatePagedMemory(os, |
| Flag, |
| &node->Virtual.bytes, |
| &node->Virtual.gid, |
| &node->Virtual.physical)); |
| |
| /* Calculate required GPU page (4096) count. */ |
| /* Assume start address is 4096 aligned. */ |
| node->Virtual.pageCount = (gctSIZE_T)(((gctUINT64)node->Virtual.bytes + (4096 - 1)) >> 12); |
| |
| /* Return pointer to the gcuVIDMEM_NODE union. */ |
| *Node = node; |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Created virtual node 0x%x for %u bytes @ 0x%x", |
| node, Bytes, node->Virtual.physical); |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Node=0x%x", *Node); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Roll back. */ |
| if (node != gcvNULL) |
| { |
| /* Free the structure. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _AddToBlockList( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_BLOCK VidMemBlock |
| ) |
| { |
| VidMemBlock->next = Kernel->vidMemBlock; |
| Kernel->vidMemBlock = VidMemBlock; |
| |
| return gcvSTATUS_OK; |
| } |
| |
| static gceSTATUS |
| _RemoveFromBlockList( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_BLOCK VidMemBlock |
| ) |
| { |
| gckVIDMEM_BLOCK vidMemBlock; |
| gckVIDMEM_BLOCK previous = gcvNULL; |
| gceHARDWARE_TYPE hwType; |
| |
| gcmkVERIFY_OK( |
| gckKERNEL_GetHardwareType(Kernel, |
| &hwType)); |
| |
| for (vidMemBlock = Kernel->vidMemBlock; vidMemBlock != gcvNULL; vidMemBlock = vidMemBlock->next) |
| { |
| if (vidMemBlock->addresses[hwType] == VidMemBlock->addresses[hwType]) |
| { |
| if (previous) |
| { |
| previous->next = vidMemBlock->next; |
| } |
| else |
| { |
| Kernel->vidMemBlock = vidMemBlock->next; |
| } |
| vidMemBlock->next = gcvNULL; |
| |
| break; |
| } |
| previous = vidMemBlock; |
| } |
| |
| return gcvSTATUS_OK; |
| } |
| |
| static gckVIDMEM_BLOCK |
| _FindFreeBlock( |
| IN gckKERNEL Kernel, |
| IN gctSIZE_T Bytes |
| ) |
| { |
| gckVIDMEM_BLOCK vidMemBlock; |
| |
| for (vidMemBlock = Kernel->vidMemBlock; vidMemBlock != gcvNULL; vidMemBlock = vidMemBlock->next) |
| { |
| if (vidMemBlock->freeBytes >= Bytes) |
| { |
| /* Found the block */ |
| break; |
| } |
| } |
| |
| return vidMemBlock; |
| } |
| |
| static gceSTATUS |
| _SplitVirtualChunk( |
| IN gckOS Os, |
| IN gcuVIDMEM_NODE_PTR Node, |
| IN gctSIZE_T Bytes |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gcuVIDMEM_NODE_PTR node = gcvNULL; |
| gctPOINTER pointer; |
| gctINT i; |
| |
| if ((Bytes <= 0) || (Bytes > Node->VirtualChunk.bytes)) |
| { |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| } |
| |
| /* Allocate a new gcuVIDMEM_NODE object. */ |
| gcmkONERROR(gckOS_Allocate(Os, |
| gcmSIZEOF(gcuVIDMEM_NODE), |
| &pointer)); |
| |
| node = pointer; |
| |
| /* Intialize the new gcuVIDMEM_NODE. */ |
| node->VirtualChunk.offset = Node->VirtualChunk.offset + Bytes; |
| node->VirtualChunk.bytes = Node->VirtualChunk.bytes - Bytes; |
| node->VirtualChunk.parent = Node->VirtualChunk.parent; |
| node->VirtualChunk.kernel = Node->VirtualChunk.kernel; |
| node->VirtualChunk.kvaddr = gcvNULL; |
| |
| for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) |
| { |
| node->VirtualChunk.lockeds[i] = 0; |
| } |
| |
| /* Insert chunk behind specified chunk. */ |
| node->VirtualChunk.next = Node->VirtualChunk.next; |
| node->VirtualChunk.prev = Node; |
| Node->VirtualChunk.next = node->VirtualChunk.next->VirtualChunk.prev = node; |
| |
| /* Insert free chunk behind specified chunk. */ |
| node->VirtualChunk.nextFree = Node->VirtualChunk.nextFree; |
| node->VirtualChunk.prevFree = Node; |
| Node->VirtualChunk.nextFree = node->VirtualChunk.nextFree->VirtualChunk.prevFree = node; |
| |
| /* Adjust size of specified chunk. */ |
| Node->VirtualChunk.bytes = Bytes; |
| |
| OnError: |
| return status; |
| } |
| |
| static gceSTATUS |
| _MergeVirtualChunk( |
| IN gckOS Os, |
| IN gcuVIDMEM_NODE_PTR Node |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node; |
| gceSTATUS status = gcvSTATUS_OK; |
| |
| node = Node->VirtualChunk.next; |
| |
| if (Node->VirtualChunk.offset + Node->VirtualChunk.bytes != node->VirtualChunk.offset) |
| { |
| /* Corrupted heap. */ |
| gcmkASSERT( |
| Node->VirtualChunk.offset + Node->VirtualChunk.bytes == node->VirtualChunk.offset); |
| |
| return gcvSTATUS_HEAP_CORRUPTED; |
| } |
| |
| /* Merge. */ |
| Node->VirtualChunk.bytes += node->VirtualChunk.bytes; |
| |
| /* Unlink next node from linked list. */ |
| Node->VirtualChunk.next = node->VirtualChunk.next; |
| Node->VirtualChunk.nextFree = node->VirtualChunk.nextFree; |
| |
| Node->VirtualChunk.next->VirtualChunk.prev = |
| Node->VirtualChunk.nextFree->VirtualChunk.prevFree = Node; |
| |
| /* Free the next node. */ |
| status = gcmkOS_SAFE_FREE(Os, node); |
| return status; |
| } |
| |
| static gctBOOL |
| _IsVidMemBlockFree( |
| IN gckVIDMEM_BLOCK VidMemBlock |
| ) |
| { |
| return (VidMemBlock->freeBytes == VidMemBlock->bytes); |
| } |
| |
| static gcuVIDMEM_NODE_PTR |
| _FindVirtualChunkNode( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_BLOCK VidMemBlock, |
| IN gctSIZE_T Bytes |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node; |
| |
| if (VidMemBlock->node.VirtualChunk.nextFree == gcvNULL) |
| { |
| /* No free chunk left. */ |
| return gcvNULL; |
| } |
| |
| for (node = VidMemBlock->node.VirtualChunk.nextFree; |
| node->VirtualChunk.bytes != 0; |
| node = node->VirtualChunk.nextFree) |
| { |
| if (node->VirtualChunk.bytes >= Bytes) |
| { |
| /* Got it. */ |
| return node; |
| } |
| } |
| |
| return gcvNULL; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** _ConvertPhysical |
| ** |
| ** Convert CPU physical to GPU address for video node. |
| ** |
| ** INPUT: |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** gcuVIDMEM_NODE_PTR Node |
| ** Pointer to a gcuVIDMEM_NODE union. |
| ** |
| ** gceCORE Core |
| ** Id of current GPU. |
| ** |
| ** gctPHYS_ADDR_T PhysicalAddress |
| ** CPU physical address |
| ** |
| ** OUTPUT: |
| ** gctUINT32 * Address |
| ** A pointer hold the GPU address. |
| */ |
| static gceSTATUS |
| _ConvertPhysical( |
| IN gckKERNEL Kernel, |
| IN gceCORE Core, |
| IN gcuVIDMEM_NODE_PTR Node, |
| IN gckVIDMEM_BLOCK VidMemBlock, |
| IN gctPHYS_ADDR_T PhysicalAddress, |
| OUT gctUINT32 * Address |
| ) |
| { |
| gceSTATUS status; |
| gctUINT64 physical = 0; |
| |
| gcmkHEADER_ARG("Node=0x%X", Node); |
| |
| if ((Node && !Node->Virtual.contiguous) || |
| (VidMemBlock && !VidMemBlock->contiguous)) |
| { |
| /* non-contiguous, mapping is required. */ |
| status = gcvSTATUS_NOT_SUPPORTED; |
| goto OnError; |
| } |
| |
| if ((Node && Node->Virtual.secure) || |
| (VidMemBlock && VidMemBlock->secure)) |
| { |
| /* Secure, mapping is forced. */ |
| status = gcvSTATUS_NOT_SUPPORTED; |
| goto OnError; |
| } |
| |
| /* Convert to GPU physical address. */ |
| gckOS_CPUPhysicalToGPUPhysical(Kernel->os, PhysicalAddress, &physical); |
| |
| |
| if ((physical > gcvMAXUINT32) || |
| (Node && (physical + Node->Virtual.bytes - 1 > gcvMAXUINT32)) || |
| (VidMemBlock && (physical + VidMemBlock->bytes - 1 > gcvMAXUINT32))) |
| { |
| /* Above 4G (32bit), mapping is required currently. */ |
| status = gcvSTATUS_NOT_SUPPORTED; |
| goto OnError; |
| } |
| |
| if (!gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_MMU)) |
| { |
| if (physical < Kernel->hardware->baseAddress) |
| { |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| } |
| |
| /* Subtract baseAddress to get a GPU address used for programming. */ |
| physical -= Kernel->hardware->baseAddress; |
| |
| /* 2G upper is virtual space, better to move to gckHARDWARE section. */ |
| if (physical + Node->Virtual.bytes > 0x80000000) |
| { |
| /* End is above 2G, ie virtual space. */ |
| status = gcvSTATUS_NOT_SUPPORTED; |
| goto OnError; |
| } |
| |
| *Address = (gctUINT32)physical; |
| |
| gcmkFOOTER_ARG("*Address=0x%X", *Address); |
| return gcvSTATUS_OK; |
| } |
| else |
| { |
| gctBOOL flatMapped; |
| |
| gcmkONERROR(gckMMU_IsFlatMapped(Kernel->mmu, physical, gcvINVALID_ADDRESS, &flatMapped)); |
| |
| if (!flatMapped) |
| { |
| status = gcvSTATUS_NOT_SUPPORTED; |
| goto OnError; |
| } |
| |
| *Address = (gctUINT32)physical; |
| |
| gcmkFOOTER_ARG("*Address=0x%X", *Address); |
| return gcvSTATUS_OK; |
| } |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| gckVIDMEM_MapVidMemBlock( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_BLOCK VidMemBlock |
| ) |
| { |
| gceSTATUS status; |
| gckOS os = Kernel->os; |
| gctPHYS_ADDR_T physAddr; |
| gceHARDWARE_TYPE hwType; |
| |
| gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p", Kernel, VidMemBlock); |
| |
| gcmkVERIFY_OK( |
| gckKERNEL_GetHardwareType(Kernel, |
| &hwType)); |
| |
| gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); |
| gcmkASSERT(VidMemBlock->pageCount > 0); |
| |
| gcmkONERROR( |
| gckOS_GetPhysicalFromHandle(os, |
| VidMemBlock->physical, |
| 0, |
| &physAddr)); |
| |
| status = _ConvertPhysical(Kernel, |
| Kernel->core, |
| gcvNULL, |
| VidMemBlock, |
| physAddr, |
| &VidMemBlock->addresses[hwType]); |
| if (gcmIS_ERROR(status)) |
| { |
| /* Allocate pages inside the MMU. */ |
| gcmkONERROR( |
| gckMMU_AllocatePagesEx(Kernel->mmu, |
| VidMemBlock->pageCount, |
| VidMemBlock->type, |
| gcvPAGE_TYPE_1M, |
| VidMemBlock->secure, |
| &VidMemBlock->pageTables[hwType], |
| &VidMemBlock->addresses[hwType])); |
| |
| if (VidMemBlock->onFault != gcvTRUE) |
| { |
| /* Map the pages. */ |
| gcmkONERROR( |
| gckOS_Map1MPages(os, |
| Kernel->core, |
| VidMemBlock->physical, |
| VidMemBlock->pageCount, |
| VidMemBlock->addresses[hwType], |
| VidMemBlock->pageTables[hwType], |
| gcvTRUE, |
| VidMemBlock->type)); |
| } |
| |
| gcmkONERROR(gckMMU_Flush(Kernel->mmu, VidMemBlock->type)); |
| |
| /* Calculate the GPU virtual address. */ |
| VidMemBlock->addresses[hwType] |= (gctUINT32) (physAddr & ((1 << 20) - 1)); |
| } |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Mapped video memory block 0x%x to 0x%08X", |
| VidMemBlock, |
| VidMemBlock->addresses[hwType]); |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (VidMemBlock->pageTables[hwType] != gcvNULL) |
| { |
| /* Free the pages from the MMU. */ |
| gcmkVERIFY_OK( |
| gckMMU_FreePages(Kernel->mmu, |
| VidMemBlock->secure, |
| gcvPAGE_TYPE_1M, |
| VidMemBlock->addresses[hwType], |
| VidMemBlock->pageTables[hwType], |
| VidMemBlock->pageCount)); |
| |
| VidMemBlock->pageTables[hwType] = gcvNULL; |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _UnmapVidMemBlock( |
| IN gckMMU Mmu, |
| IN gceHARDWARE_TYPE HwType, |
| IN gckVIDMEM_BLOCK VidMemBlock |
| ) |
| { |
| gceSTATUS status; |
| |
| gcmkHEADER_ARG("Mmu=%p VidMemBlock=%p", Mmu, VidMemBlock); |
| |
| gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); |
| |
| if (VidMemBlock->pageTables[HwType] != gcvNULL) |
| { |
| /* Free the pages from the MMU. */ |
| gcmkONERROR( |
| gckMMU_FreePages(Mmu, |
| VidMemBlock->secure, |
| gcvPAGE_TYPE_1M, |
| VidMemBlock->addresses[HwType], |
| VidMemBlock->pageTables[HwType], |
| VidMemBlock->pageCount)); |
| |
| VidMemBlock->pageTables[HwType] = gcvNULL; |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| gckVIDMEM_BLOCK_Construct( |
| IN gckKERNEL Kernel, |
| IN gctSIZE_T BlockSize, |
| IN gceVIDMEM_TYPE Type, |
| IN gctUINT32 Flag, |
| OUT gckVIDMEM_BLOCK * VidMemBlock |
| ) |
| { |
| gceSTATUS status; |
| gckVIDMEM_BLOCK vidMemBlock = gcvNULL; |
| gcuVIDMEM_NODE_PTR node = gcvNULL; |
| gckOS os = Kernel->os; |
| gctPOINTER pointer; |
| gctINT i; |
| |
| gcmkHEADER_ARG("Kernel=0x%x BlockSize=%lu Type=%x Flag=%x", Kernel, BlockSize, Type); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_ARGUMENT(BlockSize > 0); |
| gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); |
| |
| /* Allocate an gckVIDMEM_BLOCK object. */ |
| gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_BLOCK), &pointer)); |
| |
| vidMemBlock = pointer; |
| |
| /* Initialize the gckVIDMEM_BLOCK object. */ |
| vidMemBlock->object.type = gcvOBJ_VIDMEM_BLOCK; |
| vidMemBlock->os = os; |
| vidMemBlock->bytes = BlockSize; |
| vidMemBlock->freeBytes = BlockSize; |
| /* 1M page count. */ |
| vidMemBlock->pageCount = (gctUINT32)(BlockSize >> 20); |
| vidMemBlock->type = Type; |
| vidMemBlock->contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS; |
| vidMemBlock->secure = (Flag & gcvALLOC_FLAG_SECURITY) != 0; |
| vidMemBlock->onFault = (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT) != 0; |
| vidMemBlock->mutex = gcvNULL; |
| vidMemBlock->physical = gcvNULL; |
| |
| for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) |
| { |
| vidMemBlock->pageTables[i] = gcvNULL; |
| } |
| |
| /* Allocate the mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &vidMemBlock->mutex)); |
| |
| /* Allocate one gcuVIDMEM_NODE union. */ |
| gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer)); |
| |
| node = pointer; |
| |
| if (!vidMemBlock->contiguous) |
| { |
| Flag |= gcvALLOC_FLAG_1M_PAGES; |
| } |
| |
| /* Alloc 1M page size aligned memory block. */ |
| gcmkONERROR( |
| gckOS_AllocatePagedMemory(os, |
| Flag, |
| &BlockSize, |
| &vidMemBlock->gid, |
| &vidMemBlock->physical)); |
| |
| /* Map current hardware mmu table with 1M pages for this video memory block. */ |
| gcmkONERROR(gckVIDMEM_MapVidMemBlock(Kernel, vidMemBlock)); |
| |
| /* Initialize gcuVIDMEM_NODE union for virtual memory. */ |
| node->VirtualChunk.kernel = Kernel; |
| node->VirtualChunk.offset = 0; |
| node->VirtualChunk.bytes = BlockSize; |
| node->VirtualChunk.kvaddr = gcvNULL; |
| node->VirtualChunk.logical = gcvNULL; |
| node->VirtualChunk.parent = vidMemBlock; |
| |
| for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) |
| { |
| node->VirtualChunk.lockeds[i] = 0; |
| } |
| |
| /* Initialize the virtual chunk linked-list. */ |
| node->VirtualChunk.next = |
| node->VirtualChunk.prev = |
| node->VirtualChunk.nextFree = |
| node->VirtualChunk.prevFree = &vidMemBlock->node; |
| |
| vidMemBlock->node.VirtualChunk.next = |
| vidMemBlock->node.VirtualChunk.prev = |
| vidMemBlock->node.VirtualChunk.nextFree = |
| vidMemBlock->node.VirtualChunk.prevFree = node; |
| |
| vidMemBlock->node.VirtualChunk.bytes = 0; |
| |
| *VidMemBlock = vidMemBlock; |
| |
| gcmkFOOTER_ARG("*VidMemBlock=0x%x", *VidMemBlock); |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (vidMemBlock != gcvNULL) |
| { |
| if (vidMemBlock->mutex) |
| { |
| gcmkVERIFY_OK(gckOS_DeleteMutex(os, vidMemBlock->mutex)); |
| } |
| |
| if (vidMemBlock->physical) |
| { |
| gcmkVERIFY_OK(gckOS_FreePagedMemory(os, |
| vidMemBlock->physical, |
| vidMemBlock->bytes)); |
| } |
| |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, vidMemBlock)); |
| } |
| |
| if (node != gcvNULL) |
| { |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| gckVIDMEM_BLOCK_Destroy( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_BLOCK VidMemBlock |
| ) |
| { |
| gckDEVICE device = Kernel->device; |
| gctINT i; |
| |
| gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p", Kernel, VidMemBlock); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_ARGUMENT(Kernel != gcvNULL); |
| gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); |
| |
| if (VidMemBlock->physical) |
| { |
| gcmkVERIFY_OK(gckOS_FreePagedMemory(Kernel->os, |
| VidMemBlock->physical, |
| VidMemBlock->bytes)); |
| } |
| |
| for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) |
| { |
| if (VidMemBlock->pageTables[i]) |
| { |
| gcmkVERIFY_OK(_UnmapVidMemBlock(device->mmus[i], i, VidMemBlock)); |
| } |
| } |
| |
| /* Free the mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, VidMemBlock->mutex)); |
| |
| /* Free the virtual chunk. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, VidMemBlock->node.VirtualChunk.next)); |
| |
| /* Free the video memory block. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, VidMemBlock)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| static gceSTATUS |
| _AllocateVirtualChunk( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_BLOCK VidMemBlock, |
| IN gceVIDMEM_TYPE Type, |
| INOUT gctSIZE_T *Bytes, |
| OUT gcuVIDMEM_NODE_PTR *Node |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gctBOOL acquired = gcvFALSE; |
| gcuVIDMEM_NODE_PTR node; |
| gctSIZE_T bytes; |
| |
| gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p Type=%x Bytes=%zx", |
| Kernel, VidMemBlock, Type, *Bytes); |
| |
| gcmkVERIFY_ARGUMENT(Node != gcvNULL); |
| gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); |
| gcmkVERIFY_ARGUMENT(Bytes > 0); |
| gcmkVERIFY_ARGUMENT(Type < gcvVIDMEM_TYPE_COUNT); |
| |
| gcmkONERROR(gckOS_AcquireMutex(Kernel->os, VidMemBlock->mutex, gcvINFINITE)); |
| |
| acquired = gcvTRUE; |
| |
| bytes = gcmALIGN(*Bytes, 4096); |
| |
| if (bytes > VidMemBlock->freeBytes) |
| { |
| /* No enough memory. */ |
| status = gcvSTATUS_OUT_OF_MEMORY; |
| goto OnError; |
| } |
| |
| node = _FindVirtualChunkNode(Kernel, VidMemBlock, bytes); |
| if (node == gcvNULL) |
| { |
| gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY); |
| } |
| |
| if (node->VirtualChunk.bytes > bytes) |
| { |
| /* Split the chunk. */ |
| _SplitVirtualChunk(Kernel->os, node, bytes); |
| } |
| |
| /* Remove the chunk from the free list. */ |
| node->VirtualChunk.prevFree->VirtualChunk.nextFree = node->VirtualChunk.nextFree; |
| node->VirtualChunk.nextFree->VirtualChunk.prevFree = node->VirtualChunk.prevFree; |
| node->VirtualChunk.nextFree = node->VirtualChunk.prevFree = gcvNULL; |
| |
| /* Fill in the information. */ |
| node->VirtualChunk.parent = VidMemBlock; |
| |
| VidMemBlock->freeBytes -= node->VirtualChunk.bytes; |
| |
| *Bytes = bytes; |
| *Node = node; |
| |
| OnError: |
| if (acquired) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, VidMemBlock->mutex)); |
| } |
| |
| return status; |
| } |
| |
| static gceSTATUS |
| gckVIDMEM_AllocateVirtualChunk( |
| IN gckKERNEL Kernel, |
| IN gceVIDMEM_TYPE Type, |
| IN gctUINT32 Flag, |
| IN gctSIZE_T Bytes, |
| OUT gcuVIDMEM_NODE_PTR * Node |
| ) |
| { |
| gckOS os; |
| gceSTATUS status; |
| gcuVIDMEM_NODE_PTR node; |
| gckVIDMEM_BLOCK vidMemBlock = gcvNULL; |
| gctSIZE_T blockSize; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Kernel=0x%x Flag=%x Bytes=%lu", Kernel, Flag, Bytes); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); |
| gcmkVERIFY_ARGUMENT(Bytes > 0); |
| gcmkVERIFY_ARGUMENT(Node != gcvNULL); |
| |
| /* Extract the gckOS object pointer. */ |
| os = Kernel->os; |
| gcmkVERIFY_OBJECT(os, gcvOBJ_OS); |
| |
| /* Acquire the vidMem block mutex */ |
| gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->vidMemBlockMutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| /* Find the free vidmem block. */ |
| vidMemBlock = _FindFreeBlock(Kernel, Bytes); |
| if (!vidMemBlock) |
| { |
| /* Not found, construct new block. */ |
| blockSize = gcmALIGN(Bytes, gcd1M_PAGE_SIZE); |
| |
| gcmkONERROR( |
| gckVIDMEM_BLOCK_Construct(Kernel, |
| blockSize, |
| Type, |
| Flag, |
| &vidMemBlock)); |
| |
| gcmkONERROR(_AddToBlockList(Kernel, vidMemBlock)); |
| } |
| |
| /* Allocate virtual chunk node in the found block. */ |
| gcmkONERROR( |
| _AllocateVirtualChunk(Kernel, |
| vidMemBlock, |
| Type, |
| &Bytes, |
| &node)); |
| |
| /* Return pointer to the gcuVIDMEM_NODE union. */ |
| *Node = node; |
| |
| /* Release the vidMem block mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex)); |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Created virtual node 0x%x for %u bytes @ 0x%x", |
| node, Bytes, node->Virtual.physical); |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Node=0x%x", *Node); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| /* Release the vidMem block mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_Free |
| ** |
| ** Free an allocated video memory node. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** gcuVIDMEM_NODE_PTR Node |
| ** Pointer to a gcuVIDMEM_NODE object. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| static gceSTATUS |
| gckVIDMEM_Free( |
| IN gckKERNEL Kernel, |
| IN gcuVIDMEM_NODE_PTR Node |
| ) |
| { |
| gceSTATUS status; |
| gckKERNEL kernel = gcvNULL; |
| gckVIDMEM memory = gcvNULL; |
| gckVIDMEM_BLOCK vidMemBlock = gcvNULL; |
| gcuVIDMEM_NODE_PTR node; |
| gctBOOL mutexAcquired = gcvFALSE; |
| gctBOOL vbMutexAcquired = gcvFALSE; |
| gctBOOL vbListMutexAcquired = gcvFALSE; |
| gctUINT64 mappingInOne = 1; |
| |
| gcmkHEADER_ARG("Node=0x%x", Node); |
| |
| /* Verify the arguments. */ |
| if ((Node == gcvNULL) |
| || (Node->VidMem.parent == gcvNULL) |
| ) |
| { |
| /* Invalid object. */ |
| gcmkONERROR(gcvSTATUS_INVALID_OBJECT); |
| } |
| |
| vidMemBlock = Node->VirtualChunk.parent; |
| |
| /**************************** Video Memory ********************************/ |
| |
| if (Node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| /* Extract pointer to gckVIDMEM object owning the node. */ |
| memory = Node->VidMem.parent; |
| |
| /* Acquire the mutex. */ |
| gcmkONERROR( |
| gckOS_AcquireMutex(memory->os, memory->mutex, gcvINFINITE)); |
| |
| mutexAcquired = gcvTRUE; |
| |
| if (Node->VidMem.kvaddr) |
| { |
| #if gcdCAPTURE_ONLY_MODE |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Node->VidMem.kvaddr)); |
| #else |
| |
| gcmkONERROR( |
| gckOS_DestroyKernelMapping(Kernel->os, |
| Node->VidMem.parent->physical, |
| Node->VidMem.kvaddr)); |
| #endif |
| |
| Node->VidMem.kvaddr = gcvNULL; |
| } |
| |
| #ifdef __QNXNTO__ |
| /* Unmap the video memory. */ |
| if (Node->VidMem.logical != gcvNULL) |
| { |
| gckKERNEL_UnmapVideoMemory( |
| Kernel, |
| Node->VidMem.pool, |
| Node->VidMem.physical, |
| Node->VidMem.logical, |
| Node->VidMem.processID, |
| Node->VidMem.bytes |
| ); |
| |
| Node->VidMem.logical = gcvNULL; |
| } |
| |
| /* Reset. */ |
| Node->VidMem.processID = 0; |
| |
| /* Don't try to re-free an already freed node. */ |
| if ((Node->VidMem.nextFree == gcvNULL) |
| && (Node->VidMem.prevFree == gcvNULL) |
| ) |
| #endif |
| { |
| |
| /* Check if Node is already freed. */ |
| if (Node->VidMem.nextFree) |
| { |
| /* Node is alread freed. */ |
| gcmkONERROR(gcvSTATUS_INVALID_DATA); |
| } |
| |
| /* Update the number of free bytes. */ |
| memory->freeBytes += Node->VidMem.bytes; |
| |
| /* Find the next free node. */ |
| for (node = Node->VidMem.next; |
| node != gcvNULL && node->VidMem.nextFree == gcvNULL; |
| node = node->VidMem.next) ; |
| |
| if (node == gcvNULL) |
| { |
| gcmkONERROR(gcvSTATUS_INVALID_DATA); |
| } |
| |
| /* Insert this node in the free list. */ |
| Node->VidMem.nextFree = node; |
| Node->VidMem.prevFree = node->VidMem.prevFree; |
| |
| Node->VidMem.prevFree->VidMem.nextFree = |
| node->VidMem.prevFree = Node; |
| |
| /* Is the next node a free node and not the sentinel? */ |
| if ((Node->VidMem.next == Node->VidMem.nextFree) |
| && (Node->VidMem.next->VidMem.bytes != 0) |
| ) |
| { |
| /* Merge this node with the next node. */ |
| gcmkONERROR(_Merge(memory->os, node = Node)); |
| gcmkASSERT(node->VidMem.nextFree != node); |
| gcmkASSERT(node->VidMem.prevFree != node); |
| } |
| |
| /* Is the previous node a free node and not the sentinel? */ |
| if ((Node->VidMem.prev == Node->VidMem.prevFree) |
| && (Node->VidMem.prev->VidMem.bytes != 0) |
| ) |
| { |
| /* Merge this node with the previous node. */ |
| gcmkONERROR(_Merge(memory->os, node = Node->VidMem.prev)); |
| gcmkASSERT(node->VidMem.nextFree != node); |
| gcmkASSERT(node->VidMem.prevFree != node); |
| } |
| } |
| |
| gckOS_QueryOption(memory->os, "allMapInOne", &mappingInOne); |
| if (!mappingInOne) |
| { |
| gckOS_ReleaseReservedMemory(memory->os, Node->VidMem.physical); |
| Node->VidMem.physical = gcvNULL; |
| } |
| |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(memory->os, memory->mutex)); |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Node 0x%x is freed.", |
| Node); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| /* Acquire the vidMem block mutex */ |
| gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->vidMemBlockMutex, gcvINFINITE)); |
| vbListMutexAcquired = gcvTRUE; |
| |
| if (vidMemBlock) |
| { |
| gckOS os = vidMemBlock->os; |
| |
| gcmkONERROR(gckOS_AcquireMutex(os, vidMemBlock->mutex, gcvINFINITE)); |
| vbMutexAcquired = gcvTRUE; |
| kernel = Node->VirtualChunk.kernel; |
| |
| if (Node->VirtualChunk.kvaddr) |
| { |
| gcmkONERROR( |
| gckOS_DestroyKernelMapping(kernel->os, |
| vidMemBlock->physical, |
| Node->VirtualChunk.kvaddr)); |
| |
| Node->VirtualChunk.kvaddr = gcvNULL; |
| } |
| |
| /* Handle the free chunk in the linked-list */ |
| { |
| /* Check if chunk is in free list. */ |
| if (Node->VirtualChunk.nextFree) |
| { |
| /* Chunk is already freed. */ |
| gcmkONERROR(gcvSTATUS_INVALID_DATA); |
| } |
| |
| vidMemBlock->freeBytes += Node->VirtualChunk.bytes; |
| |
| /* Find the next free chunk. */ |
| for (node = Node->VirtualChunk.next; |
| node != gcvNULL && node->VirtualChunk.nextFree == gcvNULL; |
| node = node->VirtualChunk.next); |
| |
| if (node == gcvNULL) |
| { |
| gcmkONERROR(gcvSTATUS_INVALID_DATA); |
| } |
| |
| /* Insert this chunk in the free list. */ |
| Node->VirtualChunk.nextFree = node; |
| Node->VirtualChunk.prevFree = node->VirtualChunk.prevFree; |
| |
| Node->VirtualChunk.prevFree->VirtualChunk.nextFree = |
| node->VirtualChunk.prevFree = Node; |
| |
| /* Is the next chunk a free chunk. */ |
| if ((Node->VirtualChunk.next == Node->VirtualChunk.nextFree) |
| && (Node->VirtualChunk.next->VirtualChunk.bytes != 0) |
| ) |
| { |
| /* Merge this chunk with the next chunk. */ |
| gcmkONERROR(_MergeVirtualChunk(os, node = Node)); |
| gcmkASSERT(node->VirtualChunk.nextFree != node); |
| gcmkASSERT(node->VirtualChunk.prevFree != node); |
| } |
| |
| /* Is the previous chunk a free chunk. */ |
| if ((Node->VirtualChunk.prev == Node->VirtualChunk.prevFree) |
| && (Node->VirtualChunk.prev->VirtualChunk.bytes != 0) |
| ) |
| { |
| /* Merge this chunk with the previous chunk. */ |
| gcmkONERROR(_MergeVirtualChunk(os, node = Node->VirtualChunk.prev)); |
| gcmkASSERT(node->VirtualChunk.nextFree != node); |
| gcmkASSERT(node->VirtualChunk.prevFree != node); |
| } |
| } |
| |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, vidMemBlock->mutex)); |
| |
| /* Only free the vidmem block when all the chunks are freed. */ |
| if (_IsVidMemBlockFree(vidMemBlock)) |
| { |
| gcmkONERROR(_RemoveFromBlockList(kernel, vidMemBlock)); |
| |
| gcmkONERROR(gckVIDMEM_BLOCK_Destroy(kernel, vidMemBlock)); |
| } |
| } |
| |
| /* Release the vidMem block mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /*************************** Virtual Memory *******************************/ |
| |
| /* Get gckKERNEL object. */ |
| kernel = Node->Virtual.kernel; |
| |
| /* Verify the gckKERNEL object pointer. */ |
| gcmkVERIFY_OBJECT(kernel, gcvOBJ_KERNEL); |
| |
| |
| if (Node->Virtual.kvaddr) |
| { |
| gcmkVERIFY_OK( |
| gckOS_DestroyKernelMapping(kernel->os, |
| Node->Virtual.physical, |
| Node->Virtual.kvaddr)); |
| } |
| |
| /* Free the virtual memory. */ |
| gcmkVERIFY_OK(gckOS_FreePagedMemory(kernel->os, |
| Node->Virtual.physical, |
| Node->Virtual.bytes)); |
| |
| /* Delete the gcuVIDMEM_NODE union. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(kernel->os, Node)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (mutexAcquired) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex( |
| memory->os, memory->mutex |
| )); |
| } |
| |
| if (vbMutexAcquired) |
| { |
| gcmkVERIFY_OK(gckOS_ReleaseMutex( |
| vidMemBlock->os, vidMemBlock->mutex |
| )); |
| } |
| |
| if (vbListMutexAcquired) |
| { |
| gcmkVERIFY_OK(gckOS_ReleaseMutex( |
| vidMemBlock->os, Kernel->vidMemBlockMutex)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_Lock |
| ** |
| ** Lock a video memory node and return its hardware specific address. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** gcuVIDMEM_NODE_PTR Node |
| ** Pointer to a gcuVIDMEM_NODE union. |
| ** |
| ** OUTPUT: |
| ** |
| ** gctUINT32 * Address |
| ** Pointer to a variable that will hold the hardware specific address. |
| ** |
| ** gctUINT32 * PhysicalAddress |
| ** Pointer to a variable that will hold the bus address of a contiguous |
| ** video node. |
| */ |
| static gceSTATUS |
| gckVIDMEM_Lock( |
| IN gckKERNEL Kernel, |
| IN gcuVIDMEM_NODE_PTR Node, |
| OUT gctUINT32 * Address |
| ) |
| { |
| gckOS os; |
| |
| gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node); |
| |
| /* Extract the gckOS object pointer. */ |
| os = Kernel->os; |
| |
| /* Increment the lock count. */ |
| if (Node->VidMem.locked++ == 0) |
| { |
| gctUINT32 address; |
| gctUINT32 offset = (gctUINT32)Node->VidMem.offset; |
| |
| switch (Node->VidMem.pool) |
| { |
| case gcvPOOL_LOCAL_EXTERNAL: |
| address = Kernel->externalBaseAddress + offset; |
| break; |
| case gcvPOOL_LOCAL_INTERNAL: |
| address = Kernel->internalBaseAddress + offset; |
| break; |
| case gcvPOOL_INTERNAL_SRAM: |
| address = Kernel->sRAMBaseAddresses[Kernel->sRAMIndex] + offset; |
| break; |
| case gcvPOOL_EXTERNAL_SRAM: |
| address = Kernel->extSRAMBaseAddresses[Kernel->extSRAMIndex] + offset; |
| break; |
| default: |
| gcmkASSERT(Node->VidMem.pool == gcvPOOL_SYSTEM); |
| /*FALLTHRU*/ |
| case gcvPOOL_SYSTEM: |
| address = Kernel->contiguousBaseAddress + offset; |
| break; |
| } |
| |
| /* Save address. */ |
| Node->VidMem.address = address; |
| } |
| |
| *Address = Node->VidMem.address; |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Locked node 0x%x (%d) @ 0x%08X", |
| Node, |
| Node->VidMem.locked, |
| *Address); |
| |
| gcmkFOOTER_ARG("*Address=0x%08X", *Address); |
| return gcvSTATUS_OK; |
| } |
| |
| static gceSTATUS |
| gckVIDMEM_LockVirtual( |
| IN gckKERNEL Kernel, |
| IN gcuVIDMEM_NODE_PTR Node, |
| OUT gctUINT32 * Address |
| ) |
| { |
| gceSTATUS status; |
| gctPHYS_ADDR_T physicalAddress; |
| gctBOOL locked = gcvFALSE; |
| gckOS os = Kernel->os; |
| gceHARDWARE_TYPE hwType; |
| |
| gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node); |
| |
| gcmkVERIFY_OK( |
| gckKERNEL_GetHardwareType(Kernel, |
| &hwType)); |
| |
| gcmkONERROR( |
| gckOS_GetPhysicalFromHandle(os, |
| Node->Virtual.physical, |
| 0, |
| &physicalAddress)); |
| |
| |
| /* Increment the lock count. */ |
| if (Node->Virtual.lockeds[hwType]++ == 0) |
| { |
| locked = gcvTRUE; |
| |
| status = _ConvertPhysical( |
| Kernel, |
| Kernel->core, |
| Node, |
| gcvNULL, |
| physicalAddress, |
| &Node->Virtual.addresses[hwType] |
| ); |
| |
| if (gcmIS_ERROR(status)) |
| { |
| /* Do GPU address mapping. */ |
| #if gcdSECURITY |
| gctPHYS_ADDR physicalArrayPhysical; |
| gctPOINTER physicalArrayLogical; |
| |
| gcmkONERROR(gckOS_AllocatePageArray( |
| os, |
| Node->Virtual.physical, |
| Node->Virtual.pageCount, |
| &physicalArrayLogical, |
| &physicalArrayPhysical |
| )); |
| |
| gcmkONERROR(gckKERNEL_SecurityMapMemory( |
| Kernel, |
| physicalArrayLogical, |
| Node->Virtual.pageCount, |
| &Node->Virtual.addresses[hwType] |
| )); |
| |
| gcmkONERROR(gckOS_FreeNonPagedMemory( |
| os, |
| physicalArrayPhysical, |
| physicalArrayLogical, |
| 1 |
| )); |
| #else |
| { |
| /* Allocate pages inside the MMU. */ |
| gcmkONERROR( |
| gckMMU_AllocatePagesEx(Kernel->mmu, |
| Node->Virtual.pageCount, |
| Node->Virtual.type, |
| gcvPAGE_TYPE_4K, |
| Node->Virtual.secure, |
| &Node->Virtual.pageTables[hwType], |
| &Node->Virtual.addresses[hwType])); |
| } |
| |
| if (Node->Virtual.onFault != gcvTRUE) |
| { |
| #if gcdENABLE_TRUST_APPLICATION |
| if (Kernel->hardware->options.secureMode == gcvSECURE_IN_TA) |
| { |
| gcmkONERROR(gckKERNEL_MapInTrustApplicaiton( |
| Kernel, |
| Node->Virtual.logical, |
| Node->Virtual.physical, |
| Node->Virtual.addresses[hwType], |
| Node->Virtual.pageCount |
| )); |
| } |
| else |
| #endif |
| { |
| gcmkDUMP(os, "#[mmu: dynamic mapping: address=0x%08X pageCount=%lu]", |
| Node->Virtual.addresses[hwType], |
| (unsigned long)Node->Virtual.pageCount); |
| |
| /* Map the pages. */ |
| gcmkONERROR(gckOS_MapPagesEx(os, |
| Kernel->core, |
| Node->Virtual.physical, |
| Node->Virtual.pageCount, |
| Node->Virtual.addresses[hwType], |
| Node->Virtual.pageTables[hwType], |
| gcvTRUE, |
| Node->Virtual.type)); |
| } |
| } |
| |
| { |
| gcmkONERROR(gckMMU_Flush(Kernel->mmu, Node->Virtual.type)); |
| } |
| #endif |
| |
| /* GPU MMU page size is fixed at 4096 now. */ |
| Node->Virtual.addresses[hwType] |= (gctUINT32)physicalAddress & (4096 - 1); |
| } |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Mapped virtual node 0x%x to 0x%08X", |
| Node, |
| Node->Virtual.addresses[hwType]); |
| } |
| |
| /* Return hardware address. */ |
| *Address = Node->Virtual.addresses[hwType]; |
| |
| gcmkFOOTER_ARG("*Address=0x%08X", *Address); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (locked) |
| { |
| if (Node->Virtual.pageTables[hwType] != gcvNULL) |
| { |
| { |
| /* Free the pages from the MMU. */ |
| gcmkVERIFY_OK( |
| gckMMU_FreePages(Kernel->mmu, |
| Node->Virtual.secure, |
| gcvPAGE_TYPE_4K, |
| Node->Virtual.addresses[hwType], |
| Node->Virtual.pageTables[hwType], |
| Node->Virtual.pageCount)); |
| } |
| |
| Node->Virtual.pageTables[hwType] = gcvNULL; |
| } |
| |
| Node->Virtual.lockeds[hwType]--; |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| gckVIDMEM_LockVirtualChunk( |
| IN gckKERNEL Kernel, |
| IN gcuVIDMEM_NODE_PTR Node, |
| OUT gctUINT32 * Address |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gckVIDMEM_BLOCK vidMemBlock = Node->VirtualChunk.parent; |
| gceHARDWARE_TYPE hwType; |
| |
| gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node); |
| |
| gcmkVERIFY_OK( |
| gckKERNEL_GetHardwareType(Kernel, |
| &hwType)); |
| |
| gcmkASSERT(vidMemBlock != gcvNULL); |
| |
| /* Increment the lock count. */ |
| if (Node->VirtualChunk.lockeds[hwType]++ == 0) |
| { |
| if (!vidMemBlock->pageTables[hwType]) |
| { |
| /* Map current hardware mmu table with 1M pages for this video memory block. */ |
| gcmkONERROR(gckVIDMEM_MapVidMemBlock(Kernel, vidMemBlock)); |
| } |
| |
| Node->VirtualChunk.addresses[hwType] = vidMemBlock->addresses[hwType] |
| + (gctUINT32)Node->VirtualChunk.offset; |
| } |
| |
| /* Return hardware address. */ |
| *Address = Node->VirtualChunk.addresses[hwType]; |
| |
| gcmkFOOTER_ARG("*Address=0x%08X", *Address); |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_Unlock |
| ** |
| ** Unlock a video memory node. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** gcuVIDMEM_NODE_PTR Node |
| ** Pointer to a locked gcuVIDMEM_NODE union. |
| ** |
| ** gctBOOL * Asynchroneous |
| ** Pointer to a variable specifying whether the surface should be |
| ** unlocked asynchroneously or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** gctBOOL * Asynchroneous |
| ** Pointer to a variable receiving the number of bytes used in the |
| ** command buffer specified by 'Commands'. If gcvNULL, there is no |
| ** command buffer. |
| */ |
| static gceSTATUS |
| gckVIDMEM_Unlock( |
| IN gckKERNEL Kernel, |
| IN gcuVIDMEM_NODE_PTR Node, |
| IN OUT gctBOOL * Asynchroneous |
| ) |
| { |
| gceSTATUS status; |
| |
| gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d", |
| Node, gcmOPT_VALUE(Asynchroneous)); |
| |
| if (Node->VidMem.locked <= 0) |
| { |
| /* The surface was not locked. */ |
| gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED); |
| } |
| |
| if (Asynchroneous != gcvNULL) |
| { |
| /* Schedule an event to sync with GPU. */ |
| *Asynchroneous = gcvTRUE; |
| } |
| else |
| { |
| /* Decrement the lock count. */ |
| Node->VidMem.locked--; |
| } |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Unlocked node %p (%d)", |
| Node, |
| Node->VidMem.locked); |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous)); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| gckVIDMEM_UnlockVirtual( |
| IN gckKERNEL Kernel, |
| IN gcuVIDMEM_NODE_PTR Node, |
| IN OUT gctBOOL * Asynchroneous |
| ) |
| { |
| gceSTATUS status; |
| gceHARDWARE_TYPE hwType; |
| |
| gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d", |
| Node, gcmOPT_VALUE(Asynchroneous)); |
| |
| gcmkVERIFY_OK( |
| gckKERNEL_GetHardwareType(Kernel, |
| &hwType)); |
| |
| if (Asynchroneous != gcvNULL) |
| { |
| /* Schedule the surface to be unlocked. */ |
| *Asynchroneous = gcvTRUE; |
| } |
| else |
| { |
| if (Node->Virtual.lockeds[hwType] == 0) |
| { |
| gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED); |
| } |
| |
| /* Decrement lock count. */ |
| --Node->Virtual.lockeds[hwType]; |
| |
| /* See if we can unlock the resources. */ |
| if (Node->Virtual.lockeds[hwType] == 0) |
| { |
| gctUINT32 address; |
| |
| /* Adjust address to page aligned for underlying functions. */ |
| address = Node->Virtual.addresses[hwType] & ~(4096 - 1); |
| |
| #if gcdSECURITY |
| if (Node->Virtual.addresses[hwType] > 0x80000000) |
| { |
| gcmkONERROR(gckKERNEL_SecurityUnmapMemory( |
| Kernel, |
| address, |
| Node->Virtual.pageCount |
| )); |
| } |
| #else |
| /* Free the page table. */ |
| if (Node->Virtual.pageTables[hwType] != gcvNULL) |
| { |
| { |
| gcmkONERROR( |
| gckMMU_FreePages(Kernel->mmu, |
| Node->Virtual.secure, |
| gcvPAGE_TYPE_4K, |
| address, |
| Node->Virtual.pageTables[hwType], |
| Node->Virtual.pageCount)); |
| } |
| |
| gcmkONERROR(gckOS_UnmapPages( |
| Kernel->os, |
| Node->Virtual.pageCount, |
| address |
| )); |
| |
| /* Mark page table as freed. */ |
| Node->Virtual.pageTables[hwType] = gcvNULL; |
| } |
| #endif |
| } |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Unmapped virtual node %p from 0x%08X", |
| Node, Node->Virtual.addresses[hwType]); |
| } |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous)); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| gckVIDMEM_UnlockVirtualChunk( |
| IN gckKERNEL Kernel, |
| IN gcuVIDMEM_NODE_PTR Node, |
| IN OUT gctBOOL * Asynchroneous |
| ) |
| { |
| gceSTATUS status; |
| gceHARDWARE_TYPE hwType; |
| |
| gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d", |
| Node, gcmOPT_VALUE(Asynchroneous)); |
| |
| gcmkVERIFY_OK( |
| gckKERNEL_GetHardwareType(Kernel, |
| &hwType)); |
| |
| if (Asynchroneous != gcvNULL) |
| { |
| /* Schedule an event to sync with GPU. */ |
| *Asynchroneous = gcvTRUE; |
| } |
| else |
| { |
| if (Node->VirtualChunk.lockeds[hwType] == 0) |
| { |
| /* The surface was not locked. */ |
| gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED); |
| } |
| |
| /* Unmap and free pages when video memory free. */ |
| |
| /* Decrement the lock count. */ |
| --Node->VirtualChunk.lockeds[hwType]; |
| } |
| |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, |
| "Unlocked node %p (%d)", |
| Node, |
| Node->VirtualChunk.lockeds[hwType]); |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous)); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_HANDLE_Allocate |
| ** |
| ** Allocate a handle for a gckVIDMEM_NODE object. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** gckVIDMEM_NODE Node |
| ** Pointer to a gckVIDMEM_NODE object. |
| ** |
| ** OUTPUT: |
| ** |
| ** gctUINT32 * Handle |
| ** Pointer to a variable receiving a handle represent this |
| ** gckVIDMEM_NODE in userspace. |
| */ |
| gceSTATUS |
| gckVIDMEM_HANDLE_Allocate( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE Node, |
| OUT gctUINT32 * Handle |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 processID = 0; |
| gctPOINTER pointer = gcvNULL; |
| gctPOINTER handleDatabase = gcvNULL; |
| gctPOINTER mutex = gcvNULL; |
| gctUINT32 handle = 0; |
| gckVIDMEM_HANDLE handleObject = gcvNULL; |
| gckOS os = Kernel->os; |
| |
| gcmkHEADER_ARG("Kernel=0x%X, Node=0x%X", Kernel, Node); |
| |
| gcmkVERIFY_OBJECT(os, gcvOBJ_OS); |
| |
| /* Allocate a gckVIDMEM_HANDLE object. */ |
| gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_HANDLE), &pointer)); |
| |
| gcmkVERIFY_OK(gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsVIDMEM_HANDLE))); |
| |
| handleObject = pointer; |
| |
| gcmkONERROR(gckOS_AtomConstruct(os, &handleObject->reference)); |
| |
| /* Set default reference count to 1. */ |
| gckOS_AtomSet(os, handleObject->reference, 1); |
| |
| gcmkVERIFY_OK(gckOS_GetProcessID(&processID)); |
| |
| gcmkONERROR( |
| gckKERNEL_FindHandleDatbase(Kernel, |
| processID, |
| &handleDatabase, |
| &mutex)); |
| |
| /* Allocate a handle for this object. */ |
| gcmkONERROR( |
| gckKERNEL_AllocateIntegerId(handleDatabase, handleObject, &handle)); |
| |
| handleObject->node = Node; |
| handleObject->handle = handle; |
| |
| *Handle = handle; |
| |
| gcmkFOOTER_ARG("*Handle=%d", *Handle); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (handleObject != gcvNULL) |
| { |
| if (handleObject->reference != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckOS_AtomDestroy(os, handleObject->reference)); |
| } |
| |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, handleObject)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_HANDLE_Reference( |
| IN gckKERNEL Kernel, |
| IN gctUINT32 ProcessID, |
| IN gctUINT32 Handle |
| ) |
| { |
| gceSTATUS status; |
| gckVIDMEM_HANDLE handleObject = gcvNULL; |
| gctPOINTER database = gcvNULL; |
| gctPOINTER mutex = gcvNULL; |
| gctINT32 oldValue = 0; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Handle=%d PrcoessID=%d", Handle, ProcessID); |
| |
| gcmkONERROR( |
| gckKERNEL_FindHandleDatbase(Kernel, ProcessID, &database, &mutex)); |
| |
| gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| /* Translate handle to gckVIDMEM_HANDLE object. */ |
| gcmkONERROR( |
| gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject)); |
| |
| /* Increase the reference count. */ |
| gckOS_AtomIncrement(Kernel->os, handleObject->reference, &oldValue); |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| acquired = gcvFALSE; |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_HANDLE_Dereference( |
| IN gckKERNEL Kernel, |
| IN gctUINT32 ProcessID, |
| IN gctUINT32 Handle |
| ) |
| { |
| gceSTATUS status; |
| gctPOINTER handleDatabase = gcvNULL; |
| gctPOINTER mutex = gcvNULL; |
| gctINT32 oldValue = 0; |
| gckVIDMEM_HANDLE handleObject = gcvNULL; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Handle=%d PrcoessID=%d", Handle, ProcessID); |
| |
| gcmkONERROR( |
| gckKERNEL_FindHandleDatbase(Kernel, |
| ProcessID, |
| &handleDatabase, |
| &mutex)); |
| |
| gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| /* Translate handle to gckVIDMEM_HANDLE. */ |
| gcmkONERROR( |
| gckKERNEL_QueryIntegerId(handleDatabase, Handle, (gctPOINTER *)&handleObject)); |
| |
| gckOS_AtomDecrement(Kernel->os, handleObject->reference, &oldValue); |
| |
| if (oldValue == 1) |
| { |
| /* Remove handle from database if this is the last reference. */ |
| gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(handleDatabase, Handle)); |
| } |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| acquired = gcvFALSE; |
| |
| if (oldValue == 1) |
| { |
| gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, handleObject->reference)); |
| gcmkOS_SAFE_FREE(Kernel->os, handleObject); |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_HANDLE_Lookup( |
| IN gckKERNEL Kernel, |
| IN gctUINT32 ProcessID, |
| IN gctUINT32 Handle, |
| OUT gckVIDMEM_NODE * Node |
| ) |
| { |
| gceSTATUS status; |
| gckVIDMEM_HANDLE handleObject = gcvNULL; |
| gckVIDMEM_NODE node = gcvNULL; |
| gctPOINTER database = gcvNULL; |
| gctPOINTER mutex = gcvNULL; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Kernel=0x%X ProcessID=%d Handle=%d", |
| Kernel, ProcessID, Handle); |
| |
| gcmkONERROR( |
| gckKERNEL_FindHandleDatbase(Kernel, ProcessID, &database, &mutex)); |
| |
| gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| gcmkONERROR( |
| gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject)); |
| |
| node = handleObject->node; |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| acquired = gcvFALSE; |
| |
| *Node = node; |
| |
| gcmkFOOTER_ARG("*Node=%d", *Node); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_HANDLE_Lookup2( |
| IN gckKERNEL Kernel, |
| IN gcsDATABASE_PTR Database, |
| IN gctUINT32 Handle, |
| OUT gckVIDMEM_NODE * Node |
| ) |
| { |
| gceSTATUS status; |
| gckVIDMEM_HANDLE handleObject = gcvNULL; |
| gckVIDMEM_NODE node = gcvNULL; |
| gctPOINTER database = gcvNULL; |
| gctPOINTER mutex = gcvNULL; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Kernel=0x%X Database=%p Handle=%d", |
| Kernel, Database, Handle); |
| |
| database = Database->handleDatabase; |
| mutex = Database->handleDatabaseMutex; |
| |
| gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| gcmkONERROR( |
| gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject)); |
| |
| node = handleObject->node; |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| acquired = gcvFALSE; |
| |
| *Node = node; |
| |
| gcmkFOOTER_ARG("*Node=%d", *Node); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| |
| static gceSTATUS |
| gckVIDMEM_NODE_Construct( |
| IN gckKERNEL Kernel, |
| IN gcuVIDMEM_NODE_PTR VideoNode, |
| IN gceVIDMEM_TYPE Type, |
| IN gcePOOL Pool, |
| OUT gckVIDMEM_NODE * NodeObject |
| ) |
| { |
| gceSTATUS status; |
| gckVIDMEM_NODE node = gcvNULL; |
| gctPOINTER pointer = gcvNULL; |
| gckOS os = Kernel->os; |
| gctUINT i; |
| |
| /* Construct a node. */ |
| gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_NODE), &pointer)); |
| |
| gcmkVERIFY_OK(gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsVIDMEM_NODE))); |
| |
| node = pointer; |
| |
| node->metadata.magic = VIV_VIDMEM_METADATA_MAGIC; |
| node->metadata.ts_fd = -1; |
| #ifdef gcdANDROID |
| node->metadata.ts_address = 0; |
| #endif |
| |
| node->node = VideoNode; |
| node->kernel = Kernel; |
| node->type = Type; |
| node->pool = Pool; |
| |
| gcmkONERROR(gckOS_AtomConstruct(os, &node->reference)); |
| |
| gcmkONERROR(gckOS_CreateMutex(os, &node->mutex)); |
| |
| for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) |
| { |
| gcmkONERROR(gckOS_CreateSignal(os, gcvFALSE, &node->sync[i].signal)); |
| } |
| |
| /* Reference is 1 by default . */ |
| gckOS_AtomSet(os, node->reference, 1); |
| |
| gcmkVERIFY_OK( |
| gckOS_AcquireMutex(Kernel->os, |
| Kernel->db->videoMemListMutex, |
| gcvINFINITE)); |
| |
| /* Add into video memory node list. */ |
| gcsLIST_Add(&node->link, &Kernel->db->videoMemList); |
| |
| gcmkVERIFY_OK( |
| gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex)); |
| |
| *NodeObject = node; |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (node != gcvNULL) |
| { |
| if (node->mutex) |
| { |
| gcmkVERIFY_OK(gckOS_DeleteMutex(os, node->mutex)); |
| } |
| |
| if (node->reference != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckOS_AtomDestroy(os, node->reference)); |
| } |
| |
| for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) |
| { |
| if (node->sync[i].signal != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckOS_DestroySignal(os, node->sync[i].signal)); |
| } |
| } |
| |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node)); |
| } |
| |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_AllocateLinear( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM VideoMemory, |
| IN gcePOOL Pool, |
| IN gceVIDMEM_TYPE Type, |
| IN gctUINT32 Flag, |
| IN gctUINT32 Alignment, |
| IN gctBOOL Specified, |
| IN OUT gctSIZE_T * Bytes, |
| OUT gckVIDMEM_NODE * NodeObject |
| ) |
| { |
| gceSTATUS status; |
| gctSIZE_T bytes = *Bytes; |
| gcuVIDMEM_NODE_PTR node = gcvNULL; |
| gckVIDMEM_NODE nodeObject = gcvNULL; |
| |
| gcmkHEADER_ARG("Kernel=%p VideoMemory=%p Pool=%d Alignment=%d Type=%d *Bytes=%u", |
| Kernel, VideoMemory, Pool, Alignment, Type, bytes); |
| |
| gcmkONERROR( |
| gckVIDMEM_AllocateLinear(Kernel, |
| VideoMemory, |
| bytes, |
| Alignment, |
| Type, |
| Flag, |
| Specified, |
| &node)); |
| |
| /* Update pool. */ |
| node->VidMem.pool = Pool; |
| bytes = node->VidMem.bytes; |
| |
| /* Construct a node. */ |
| gcmkONERROR( |
| gckVIDMEM_NODE_Construct(Kernel, node, Type, Pool, &nodeObject)); |
| |
| *Bytes = bytes; |
| *NodeObject = nodeObject; |
| |
| gcmkFOOTER_ARG("*Bytes=%u *NodeObject=%p", bytes, nodeObject); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (node) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_AllocateVirtual( |
| IN gckKERNEL Kernel, |
| IN gcePOOL Pool, |
| IN gceVIDMEM_TYPE Type, |
| IN gctUINT32 Flag, |
| IN OUT gctSIZE_T * Bytes, |
| OUT gckVIDMEM_NODE * NodeObject |
| ) |
| { |
| gceSTATUS status; |
| gctSIZE_T bytes = *Bytes; |
| gcuVIDMEM_NODE_PTR node = gcvNULL; |
| gckVIDMEM_NODE nodeObject = gcvNULL; |
| |
| gcmkHEADER_ARG("Kernel=%p Pool=%d Type=%d Flag=%x *Bytes=%u", |
| Kernel, Pool, Type, Flag, bytes); |
| |
| gcmkONERROR( |
| gckVIDMEM_AllocateVirtual(Kernel, Flag, bytes, &node)); |
| |
| /* Update type. */ |
| node->Virtual.type = Type; |
| bytes = node->Virtual.bytes; |
| |
| /* Construct a node. */ |
| gcmkONERROR( |
| gckVIDMEM_NODE_Construct(Kernel, node, Type, Pool, &nodeObject)); |
| |
| *Bytes = bytes; |
| *NodeObject = nodeObject; |
| |
| gcmkFOOTER_ARG("*Bytes=%u *NodeObject=%p", bytes, nodeObject); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (node) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_AllocateVirtualChunk( |
| IN gckKERNEL Kernel, |
| IN gcePOOL Pool, |
| IN gceVIDMEM_TYPE Type, |
| IN gctUINT32 Flag, |
| IN OUT gctSIZE_T * Bytes, |
| OUT gckVIDMEM_NODE * NodeObject |
| ) |
| { |
| gceSTATUS status; |
| gctSIZE_T bytes = *Bytes; |
| gcuVIDMEM_NODE_PTR node = gcvNULL; |
| gckVIDMEM_NODE nodeObject = gcvNULL; |
| |
| gcmkHEADER_ARG("Kernel=%p Pool=%d Type=%d Flag=%x *Bytes=%u", |
| Kernel, Pool, Type, Flag, bytes); |
| |
| gcmkONERROR( |
| gckVIDMEM_AllocateVirtualChunk(Kernel, Type, Flag, bytes, &node)); |
| |
| bytes = node->VirtualChunk.bytes; |
| |
| /* Construct a node. */ |
| gcmkONERROR( |
| gckVIDMEM_NODE_Construct(Kernel, node, Type, Pool, &nodeObject)); |
| |
| *Bytes = bytes; |
| *NodeObject = nodeObject; |
| |
| gcmkFOOTER_ARG("*Bytes=%u *NodeObject=%p", bytes, nodeObject); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (node) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_Reference( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject |
| ) |
| { |
| gctINT32 oldValue; |
| gcmkHEADER_ARG("Kernel=0x%X NodeObject=0x%X", Kernel, NodeObject); |
| |
| gcmkVERIFY_ARGUMENT(NodeObject != gcvNULL); |
| |
| gckOS_AtomIncrement(Kernel->os, NodeObject->reference, &oldValue); |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_Dereference( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject |
| ) |
| { |
| gctINT32 oldValue = 0; |
| gctPOINTER database = Kernel->db->nameDatabase; |
| gctPOINTER mutex = Kernel->db->nameDatabaseMutex; |
| gctUINT i; |
| |
| gcmkHEADER_ARG("Kernel=0x%X NodeObject=0x%X", Kernel, NodeObject); |
| gcmkVERIFY_ARGUMENT(NodeObject != gcvNULL); |
| |
| gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); |
| |
| gcmkVERIFY_OK(gckOS_AtomDecrement(Kernel->os, NodeObject->reference, &oldValue)); |
| |
| if (oldValue == 1 && NodeObject->name) |
| { |
| /* Free name if exists. */ |
| gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(database, NodeObject->name)); |
| } |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| |
| if (oldValue == 1) |
| { |
| gcmkVERIFY_OK( |
| gckOS_AcquireMutex(Kernel->os, |
| Kernel->db->videoMemListMutex, |
| gcvINFINITE)); |
| |
| /* Remove from video memory node list. */ |
| gcsLIST_Del(&NodeObject->link); |
| |
| gcmkVERIFY_OK( |
| gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex)); |
| |
| /* Free gcuVIDMEM_NODE. */ |
| gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, NodeObject->node)); |
| |
| gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, NodeObject->reference)); |
| |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, NodeObject->mutex)); |
| |
| for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) |
| { |
| if (NodeObject->sync[i].signal != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckOS_DestroySignal(Kernel->os, NodeObject->sync[i].signal)); |
| } |
| } |
| |
| /* Should not cause recursive call since tsNode->tsNode should be NULL */ |
| if (NodeObject->tsNode) |
| { |
| gcmkASSERT(!NodeObject->tsNode->tsNode); |
| gckVIDMEM_NODE_Dereference(Kernel, NodeObject->tsNode); |
| } |
| |
| gcmkOS_SAFE_FREE(Kernel->os, NodeObject); |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_GetReference( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gctINT32 * ReferenceCount |
| ) |
| { |
| gctINT32 value; |
| |
| gckOS_AtomGet(Kernel->os, NodeObject->reference, &value); |
| |
| *ReferenceCount = value; |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_Lock( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gctUINT32 * Address |
| ) |
| { |
| gceSTATUS status; |
| gckOS os = Kernel->os; |
| gctBOOL acquired = gcvFALSE; |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| |
| gcmkHEADER_ARG("NodeObject=%p", NodeObject); |
| |
| /* Grab the mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| gcmkONERROR(gckVIDMEM_Lock(Kernel, node, Address)); |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| gcmkONERROR(gckVIDMEM_LockVirtualChunk(Kernel, node, Address)); |
| } |
| else |
| { |
| gcmkONERROR(gckVIDMEM_LockVirtual(Kernel, node, Address)); |
| } |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| |
| gcmkFOOTER_ARG("*Address=0x%08X", *Address); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_Unlock( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| IN gctUINT32 ProcessID, |
| IN OUT gctBOOL * Asynchroneous |
| ) |
| { |
| gceSTATUS status; |
| gckOS os = Kernel->os; |
| gctBOOL acquired = gcvFALSE; |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| |
| gcmkHEADER_ARG("NodeObject=%p Asynchroneous=%p", NodeObject, Asynchroneous); |
| |
| /* Grab the mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| gcmkONERROR(gckVIDMEM_Unlock(Kernel, node, Asynchroneous)); |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| gcmkONERROR(gckVIDMEM_UnlockVirtualChunk(Kernel, node, Asynchroneous)); |
| } |
| else |
| { |
| gcmkONERROR(gckVIDMEM_UnlockVirtual(Kernel, node, Asynchroneous)); |
| } |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| |
| gcmkFOOTER_ARG("*Asynchroneous=0x%08X", gcmOPT_VALUE(Asynchroneous)); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_CleanCache( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| IN gctSIZE_T Offset, |
| IN gctPOINTER Logical, |
| IN gctSIZE_T Bytes |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gctPHYS_ADDR physHandle = gcvNULL; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| gceSTATUS status; |
| |
| gcmkHEADER_ARG("Kernel=%p NodeObject=%d Offset=0x%llx Logical=%p Bytes=0x%llx", |
| Kernel, NodeObject, Offset, Logical, Bytes); |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| gcmkONERROR(gckOS_MemoryBarrier(Kernel->os, Logical)); |
| |
| /* Reserved pool can't be cacheable */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| physHandle = vidMemBlock->physical; |
| } |
| else |
| { |
| physHandle = node->Virtual.physical; |
| } |
| |
| gcmkONERROR(gckOS_CacheFlush( |
| Kernel->os, |
| 0, |
| physHandle, |
| Offset, |
| Logical, |
| Bytes |
| )); |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_InvalidateCache( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| IN gctSIZE_T Offset, |
| IN gctPOINTER Logical, |
| IN gctSIZE_T Bytes |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gctPHYS_ADDR physHandle = gcvNULL; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| gceSTATUS status; |
| |
| gcmkHEADER_ARG("Kernel=%p NodeObject=%d Offset=0x%llx Logical=%p Bytes=0x%llx", |
| Kernel, NodeObject, Offset, Logical, Bytes); |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| /* Reserved pool can't be cacheable */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| physHandle = vidMemBlock->physical; |
| } |
| else |
| { |
| physHandle = node->Virtual.physical; |
| } |
| |
| gcmkONERROR(gckOS_CacheInvalidate( |
| Kernel->os, |
| 0, |
| physHandle, |
| Offset, |
| Logical, |
| Bytes |
| )); |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_GetLockCount( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gctINT32 * LockCount |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| gctINT32 lockCount = 0; |
| gctINT i = 0; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| lockCount = node->VidMem.locked; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| for (; i < gcvHARDWARE_NUM_TYPES; i++) |
| { |
| lockCount += node->VirtualChunk.lockeds[i]; |
| } |
| } |
| else |
| { |
| |
| for (; i < gcvHARDWARE_NUM_TYPES; i++) |
| { |
| lockCount += node->Virtual.lockeds[i]; |
| } |
| } |
| |
| *LockCount = lockCount; |
| |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_LockCPU( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| IN gctBOOL Cacheable, |
| IN gctBOOL FromUser, |
| OUT gctPOINTER * Logical |
| ) |
| { |
| gceSTATUS status; |
| gckOS os = Kernel->os; |
| gctBOOL acquired = gcvFALSE; |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| gctPOINTER logical = gcvNULL; |
| |
| gcmkHEADER_ARG("NodeObject=%p", NodeObject); |
| |
| /* Grab the mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| if (Cacheable == gcvTRUE) |
| { |
| gcmkONERROR(gcvSTATUS_INVALID_REQUEST); |
| } |
| |
| if (FromUser) |
| { |
| #if gcdCAPTURE_ONLY_MODE |
| node->VidMem.logical = NodeObject->captureLogical; |
| #else |
| /* Map video memory pool to user space. */ |
| gcmkONERROR( |
| gckKERNEL_MapVideoMemory(Kernel, |
| gcvTRUE, |
| node->VidMem.pool, |
| node->VidMem.physical, |
| (gctUINT32)node->VidMem.offset, |
| (gctUINT32)node->VidMem.bytes, |
| &node->VidMem.logical)); |
| #endif |
| |
| logical = node->VidMem.logical; |
| } |
| else |
| { |
| /* Map video memory pool to kernel space. */ |
| if (!node->VidMem.kvaddr) |
| { |
| #if gcdCAPTURE_ONLY_MODE |
| gcmkONERROR(gckOS_Allocate(os, |
| node->VidMem.bytes, |
| &node->VidMem.kvaddr)); |
| #else |
| gcmkONERROR( |
| gckOS_CreateKernelMapping(os, |
| node->VidMem.parent->physical, |
| node->VidMem.offset, |
| node->VidMem.bytes, |
| &node->VidMem.kvaddr)); |
| #endif |
| } |
| |
| logical = node->VidMem.kvaddr; |
| } |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| if (FromUser) |
| { |
| /* Lock the entire vidmem block. */ |
| gcmkONERROR( |
| gckOS_LockPages(os, |
| vidMemBlock->physical, |
| vidMemBlock->bytes, |
| Cacheable, |
| &logical)); |
| |
| /* Get the logical with offset in block. */ |
| logical = (uint8_t *)logical + node->VirtualChunk.offset; |
| node->VirtualChunk.logical = logical; |
| } |
| else |
| { |
| /* Map once and will cancel map when free. */ |
| if (!node->VirtualChunk.kvaddr) |
| { |
| gcmkONERROR( |
| gckOS_CreateKernelMapping(os, |
| vidMemBlock->physical, |
| node->VirtualChunk.offset, |
| node->VirtualChunk.bytes, |
| &node->VirtualChunk.kvaddr)); |
| } |
| |
| logical = node->VirtualChunk.kvaddr; |
| } |
| } |
| else |
| { |
| if (FromUser) |
| { |
| gcmkONERROR( |
| gckOS_LockPages(os, |
| node->Virtual.physical, |
| node->Virtual.bytes, |
| Cacheable, |
| &logical)); |
| |
| node->Virtual.logical = logical; |
| } |
| else |
| { |
| /* Map once and will cancel map when free. */ |
| if (!node->Virtual.kvaddr) |
| { |
| gcmkONERROR( |
| gckOS_CreateKernelMapping(os, |
| node->Virtual.physical, |
| 0, |
| node->Virtual.bytes, |
| &node->Virtual.kvaddr)); |
| } |
| |
| logical = node->Virtual.kvaddr; |
| } |
| } |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| |
| *Logical = logical; |
| |
| gcmkFOOTER_ARG("*Logical=%p", logical); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_UnlockCPU( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| IN gctUINT32 ProcessID, |
| IN gctBOOL FromUser, |
| IN gctBOOL Defer |
| ) |
| { |
| gceSTATUS status; |
| gckOS os = Kernel->os; |
| gctBOOL acquired = gcvFALSE; |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| |
| gcmkHEADER_ARG("NodeObject=%p", NodeObject); |
| |
| /* Grab the mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| if (FromUser) |
| { |
| #if gcdCAPTURE_ONLY_MODE || defined __QNXNTO__ |
| /* Do nothing here. */ |
| #else |
| if (!Defer) |
| { |
| /* Unmap the video memory. */ |
| if (node->VidMem.logical != gcvNULL) |
| { |
| gckKERNEL_UnmapVideoMemory( |
| Kernel, |
| node->VidMem.pool, |
| node->VidMem.physical, |
| node->VidMem.logical, |
| node->VidMem.processID, |
| node->VidMem.bytes |
| ); |
| |
| node->VidMem.logical = gcvNULL; |
| } |
| /* Reset. */ |
| node->VidMem.processID = 0; |
| } |
| #endif |
| } |
| else |
| { |
| /* |
| * Kernel side may lock for CPU access for multiple times. Since |
| * we don't have lock counts currently, we don't cancel CPU |
| * mapping here, and will cancel at 'free' instead. |
| */ |
| /* |
| gcmkONERROR( |
| gckOS_DestroyKernelMapping(os, |
| node->VidMem.parent->physical, |
| node->VidMem.kvaddr)); |
| |
| node->VidMem.kvaddr = gcvNULL; |
| */ |
| } |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| if (FromUser) |
| { |
| gcmkONERROR( |
| gckOS_UnlockPages(os, |
| vidMemBlock->physical, |
| vidMemBlock->bytes, |
| node->VirtualChunk.logical)); |
| } |
| else |
| { |
| /* Nothing to do. */ |
| } |
| } |
| else |
| { |
| if (FromUser) |
| { |
| gcmkONERROR( |
| gckOS_UnlockPages(os, |
| node->Virtual.physical, |
| node->Virtual.bytes, |
| node->Virtual.logical)); |
| } |
| else |
| { |
| /* |
| * Kernel side may lock for CPU access for multiple times. Since |
| * we don't have lock counts currently, we don't cancel CPU |
| * mapping here, and will cancel at 'free' instead. |
| */ |
| /* |
| gcmkONERROR( |
| gckOS_DestroyKernelMapping(os, |
| node->Virtual.physical, |
| node->Virtual.kvaddr)); |
| |
| node->Virtual.kvaddr = gcvNULL; |
| */ |
| } |
| } |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_GetPhysical( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| IN gctUINT32 Offset, |
| OUT gctPHYS_ADDR_T * PhysicalAddress |
| ) |
| { |
| gceSTATUS status; |
| gckOS os = Kernel->os; |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| |
| gcmkHEADER_ARG("NodeObject=%p", NodeObject); |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| if (Offset >= node->VidMem.bytes) |
| { |
| /* Exceeds node size. */ |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| } |
| |
| *PhysicalAddress = node->VidMem.parent->physicalBase |
| + node->VidMem.offset |
| + Offset; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| if (Offset >= node->VirtualChunk.bytes) |
| { |
| /* Exceeds node size. */ |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| } |
| |
| gcmkONERROR( |
| gckOS_GetPhysicalFromHandle(os, |
| vidMemBlock->physical, |
| (gctUINT32)node->VirtualChunk.offset + Offset, |
| PhysicalAddress)); |
| } |
| else |
| { |
| if (Offset >= node->Virtual.bytes) |
| { |
| /* Exceeds node size. */ |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| } |
| |
| gcmkONERROR( |
| gckOS_GetPhysicalFromHandle(os, |
| node->Virtual.physical, |
| Offset, |
| PhysicalAddress)); |
| } |
| |
| gcmkFOOTER_ARG("*PhysicalAddress=0x%llx", *PhysicalAddress); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_GetGid( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gctUINT32 * Gid |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| *Gid = 0; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| *Gid = vidMemBlock->gid; |
| } |
| else |
| { |
| *Gid = node->Virtual.gid; |
| } |
| |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_GetSize( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gctSIZE_T * Size |
| ) |
| { |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| *Size = node->VidMem.bytes; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| *Size = node->VirtualChunk.bytes; |
| } |
| else |
| { |
| *Size = node->Virtual.bytes; |
| } |
| |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_GetType( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gceVIDMEM_TYPE * Type, |
| OUT gcePOOL * Pool |
| ) |
| { |
| if (Type) |
| { |
| *Type = NodeObject->type; |
| } |
| |
| if (Pool) |
| { |
| *Pool = NodeObject->pool; |
| } |
| |
| return gcvSTATUS_OK; |
| } |
| |
| #if defined(CONFIG_DMA_SHARED_BUFFER) |
| |
| /******************************************************************************* |
| ** |
| ** |
| ** Code for dma_buf ops |
| ** |
| ** |
| *******************************************************************************/ |
| |
| #include <linux/slab.h> |
| #include <linux/mm_types.h> |
| #include <linux/dma-buf.h> |
| |
| static struct sg_table *_dmabuf_map(struct dma_buf_attachment *attachment, |
| enum dma_data_direction direction) |
| { |
| struct sg_table *sgt = gcvNULL; |
| struct dma_buf *dmabuf = attachment->dmabuf; |
| gckVIDMEM_NODE nodeObject = dmabuf->priv; |
| gceSTATUS status = gcvSTATUS_OK; |
| |
| do |
| { |
| gcuVIDMEM_NODE_PTR node = nodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| gctPHYS_ADDR physical = gcvNULL; |
| gctSIZE_T offset = 0; |
| gctSIZE_T bytes = 0; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| physical = node->VidMem.parent->physical; |
| offset = node->VidMem.offset; |
| bytes = node->VidMem.bytes; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| physical = vidMemBlock->physical; |
| offset = node->VirtualChunk.offset; |
| bytes = node->VirtualChunk.bytes; |
| } |
| else |
| { |
| physical = node->Virtual.physical; |
| offset = 0; |
| bytes = node->Virtual.bytes; |
| } |
| |
| gcmkERR_BREAK(gckOS_MemoryGetSGT(nodeObject->kernel->os, physical, offset, bytes, (gctPOINTER*)&sgt)); |
| |
| if (dma_map_sg(attachment->dev, sgt->sgl, sgt->nents, direction) == 0) |
| { |
| sg_free_table(sgt); |
| kfree(sgt); |
| sgt = gcvNULL; |
| gcmkERR_BREAK(gcvSTATUS_GENERIC_IO); |
| } |
| } |
| while (gcvFALSE); |
| |
| return sgt; |
| } |
| |
| static void _dmabuf_unmap(struct dma_buf_attachment *attachment, |
| struct sg_table *sgt, |
| enum dma_data_direction direction) |
| { |
| dma_unmap_sg(attachment->dev, sgt->sgl, sgt->nents, direction); |
| |
| sg_free_table(sgt); |
| kfree(sgt); |
| } |
| |
| static int _dmabuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) |
| { |
| gckVIDMEM_NODE nodeObject = dmabuf->priv; |
| gcuVIDMEM_NODE_PTR node = nodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| gctPHYS_ADDR physical = gcvNULL; |
| gctSIZE_T skipPages = vma->vm_pgoff; |
| gctSIZE_T numPages = PAGE_ALIGN(vma->vm_end - vma->vm_start) >> PAGE_SHIFT; |
| gceSTATUS status = gcvSTATUS_OK; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| physical = node->VidMem.parent->physical; |
| skipPages += (node->VidMem.offset >> PAGE_SHIFT); |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| physical = vidMemBlock->physical; |
| skipPages += (node->VirtualChunk.offset >> PAGE_SHIFT); |
| } |
| else |
| { |
| physical = node->Virtual.physical; |
| } |
| |
| gcmkONERROR(gckOS_MemoryMmap(nodeObject->kernel->os, physical, skipPages, numPages, vma)); |
| |
| OnError: |
| return gcmIS_ERROR(status) ? -EINVAL : 0; |
| } |
| |
| static void _dmabuf_release(struct dma_buf *dmabuf) |
| { |
| gckVIDMEM_NODE nodeObject = dmabuf->priv; |
| |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(nodeObject->kernel, nodeObject)); |
| } |
| |
| #if LINUX_VERSION_CODE <= KERNEL_VERSION(5,5,7) |
| static void *_dmabuf_kmap(struct dma_buf *dmabuf, unsigned long offset) |
| { |
| gckVIDMEM_NODE nodeObject = dmabuf->priv; |
| gcuVIDMEM_NODE_PTR node = nodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| gctINT8_PTR kvaddr = gcvNULL; |
| gctPHYS_ADDR physical = gcvNULL; |
| gctSIZE_T bytes = 0; |
| |
| offset = (offset << PAGE_SHIFT); |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| physical = node->VidMem.parent->physical; |
| offset += node->VidMem.offset; |
| bytes = node->VidMem.bytes; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| physical = vidMemBlock->physical; |
| offset += node->VirtualChunk.offset; |
| bytes = node->VirtualChunk.bytes; |
| } |
| else |
| { |
| physical = node->Virtual.physical; |
| bytes = node->Virtual.bytes; |
| } |
| |
| if (gcmIS_SUCCESS(gckOS_CreateKernelMapping( |
| nodeObject->kernel->os, physical, 0, bytes, (gctPOINTER*)&kvaddr))) |
| { |
| kvaddr += offset; |
| } |
| |
| return (gctPOINTER)kvaddr; |
| } |
| |
| static void _dmabuf_kunmap(struct dma_buf *dmabuf, unsigned long offset, void *ptr) |
| { |
| gckVIDMEM_NODE nodeObject = dmabuf->priv; |
| gcuVIDMEM_NODE_PTR node = nodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| gctINT8_PTR kvaddr = (gctINT8_PTR)ptr - (offset << PAGE_SHIFT); |
| gctPHYS_ADDR physical = gcvNULL; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| physical = node->VidMem.parent->physical; |
| kvaddr -= node->VidMem.offset; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| physical = vidMemBlock->physical; |
| kvaddr -= node->VirtualChunk.offset; |
| } |
| else |
| { |
| physical = node->Virtual.physical; |
| } |
| |
| gcmkVERIFY_OK(gckOS_DestroyKernelMapping( |
| nodeObject->kernel->os, physical, (gctPOINTER*)&kvaddr)); |
| } |
| #endif |
| |
| static struct dma_buf_ops _dmabuf_ops = |
| { |
| .map_dma_buf = _dmabuf_map, |
| .unmap_dma_buf = _dmabuf_unmap, |
| .mmap = _dmabuf_mmap, |
| .release = _dmabuf_release, |
| #if LINUX_VERSION_CODE > KERNEL_VERSION(5,5,7) |
| # elif LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0) |
| .map = _dmabuf_kmap, |
| .unmap = _dmabuf_kunmap, |
| # elif LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) |
| .map_atomic = _dmabuf_kmap, |
| .unmap_atomic = _dmabuf_kunmap, |
| .map = _dmabuf_kmap, |
| .unmap = _dmabuf_kunmap, |
| # else |
| .kmap_atomic = _dmabuf_kmap, |
| .kunmap_atomic = _dmabuf_kunmap, |
| .kmap = _dmabuf_kmap, |
| .kunmap = _dmabuf_kunmap, |
| # endif |
| }; |
| #endif |
| |
| gceSTATUS |
| gckVIDMEM_NODE_Export( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| IN gctINT32 Flags, |
| OUT gctPOINTER *DmaBuf, |
| OUT gctINT32 *FD |
| ) |
| { |
| #if defined(CONFIG_DMA_SHARED_BUFFER) |
| gceSTATUS status = gcvSTATUS_OK; |
| struct dma_buf *dmabuf = gcvNULL; |
| |
| gcmkHEADER_ARG("Kernel=%p NodeObject=0x%x", Kernel, NodeObject); |
| |
| dmabuf = NodeObject->dmabuf; |
| if (!dmabuf) |
| { |
| gctSIZE_T bytes = 0; |
| gctPHYS_ADDR physical = gcvNULL; |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| physical = node->VidMem.parent->physical; |
| bytes = node->VidMem.bytes; |
| /* Align export size. when allocate memory from VIDMEM, the actual node size may not same with aligned size. */ |
| bytes = bytes & ~(PAGE_SIZE - 1); |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| physical = vidMemBlock->physical; |
| bytes = node->VirtualChunk.bytes; |
| } |
| else |
| { |
| physical = node->Virtual.physical; |
| bytes = node->Virtual.bytes; |
| } |
| |
| /* Donot really get SGT, just check if the allocator support GetSGT. */ |
| gcmkONERROR(gckOS_MemoryGetSGT(Kernel->os, physical, 0, 0, NULL)); |
| |
| { |
| #if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0) |
| DEFINE_DMA_BUF_EXPORT_INFO(exp_info); |
| exp_info.ops = &_dmabuf_ops; |
| exp_info.size = bytes; |
| exp_info.flags = Flags; |
| exp_info.priv = NodeObject; |
| dmabuf = dma_buf_export(&exp_info); |
| #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0) |
| dmabuf = dma_buf_export(NodeObject, &_dmabuf_ops, bytes, Flags, NULL); |
| #else |
| dmabuf = dma_buf_export(NodeObject, &_dmabuf_ops, bytes, Flags); |
| #endif |
| } |
| |
| if (IS_ERR(dmabuf)) |
| { |
| gcmkONERROR(gcvSTATUS_GENERIC_IO); |
| } |
| |
| /* Reference this gckVIDMEM_NODE object. */ |
| gckVIDMEM_NODE_Reference(Kernel, NodeObject); |
| NodeObject->dmabuf = dmabuf; |
| } |
| |
| if (DmaBuf) |
| { |
| *DmaBuf = NodeObject->dmabuf; |
| } |
| |
| if (FD) |
| { |
| gctINT fd = dma_buf_fd(dmabuf, Flags); |
| |
| if (fd < 0) |
| { |
| gcmkONERROR(gcvSTATUS_GENERIC_IO); |
| } |
| |
| *FD = fd; |
| } |
| |
| OnError: |
| gcmkFOOTER_ARG("*DmaBuf=%p *FD=0x%x", gcmOPT_POINTER(DmaBuf), gcmOPT_VALUE(FD)); |
| return status; |
| #else |
| gcmkFATAL("The kernel did NOT support CONFIG_DMA_SHARED_BUFFER"); |
| return gcvSTATUS_NOT_SUPPORTED; |
| #endif |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_Name( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gctUINT32 * Name |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 name = 0; |
| gctPOINTER database = Kernel->db->nameDatabase; |
| gctPOINTER mutex = Kernel->db->nameDatabaseMutex; |
| gctBOOL acquired = gcvFALSE; |
| gctBOOL referenced = gcvFALSE; |
| gcmkHEADER_ARG("Kernel=0x%X NodeObject=%p", Kernel, NodeObject); |
| |
| gcmkVERIFY_ARGUMENT(Name != gcvNULL); |
| |
| gcmkONERROR(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| gcmkONERROR(gckVIDMEM_NODE_Reference(Kernel, NodeObject)); |
| referenced = gcvTRUE; |
| |
| if (NodeObject->name == 0) |
| { |
| /* Name this node. */ |
| gcmkONERROR(gckKERNEL_AllocateIntegerId(database, NodeObject, &name)); |
| NodeObject->name = name; |
| } |
| else |
| { |
| name = NodeObject->name; |
| } |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| acquired = gcvFALSE; |
| |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, NodeObject)); |
| |
| *Name = name; |
| |
| gcmkFOOTER_ARG("*Name=%d", *Name); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (referenced) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, NodeObject)); |
| } |
| |
| if (acquired) |
| { |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_Import( |
| IN gckKERNEL Kernel, |
| IN gctUINT32 Name, |
| OUT gckVIDMEM_NODE * NodeObject |
| ) |
| { |
| gceSTATUS status; |
| gckVIDMEM_NODE node = gcvNULL; |
| gctPOINTER database = Kernel->db->nameDatabase; |
| gctPOINTER mutex = Kernel->db->nameDatabaseMutex; |
| gctBOOL acquired = gcvFALSE; |
| gctBOOL referenced = gcvFALSE; |
| |
| gcmkHEADER_ARG("Kernel=0x%X Name=%d", Kernel, Name); |
| |
| gcmkONERROR(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| /* Lookup in database to get the node. */ |
| gcmkONERROR(gckKERNEL_QueryIntegerId(database, Name, (gctPOINTER *)&node)); |
| |
| /* Reference the node. */ |
| gcmkONERROR(gckVIDMEM_NODE_Reference(Kernel, node)); |
| referenced = gcvTRUE; |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| acquired = gcvFALSE; |
| |
| *NodeObject = node; |
| gcmkFOOTER_ARG("*NodeObject=%p", node); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (referenced) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, node)); |
| } |
| |
| if (acquired) |
| { |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| typedef struct _gcsVIDMEM_NODE_FDPRIVATE |
| { |
| gcsFDPRIVATE base; |
| gckKERNEL kernel; |
| gckVIDMEM_NODE node; |
| } |
| gcsVIDMEM_NODE_FDPRIVATE; |
| |
| |
| static gctINT |
| _ReleaseFdPrivate( |
| gcsFDPRIVATE_PTR FdPrivate |
| ) |
| { |
| /* Cast private info. */ |
| gcsVIDMEM_NODE_FDPRIVATE * private = (gcsVIDMEM_NODE_FDPRIVATE *) FdPrivate; |
| |
| gckVIDMEM_NODE_Dereference(private->kernel, private->node); |
| gckOS_Free(private->kernel->os, private); |
| |
| return 0; |
| } |
| |
| |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_NODE_GetFd |
| ** |
| ** Attach a gckVIDMEM_NODE object to a native fd. |
| ** |
| ** OUTPUT: |
| ** |
| ** gctUINT32 * Fd |
| ** Pointer to a variable receiving a native fd from os. |
| */ |
| gceSTATUS |
| gckVIDMEM_NODE_GetFd( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gctINT * Fd |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL referenced = gcvFALSE; |
| gcsVIDMEM_NODE_FDPRIVATE * fdPrivate = gcvNULL; |
| gcmkHEADER_ARG("Kernel=0x%X NodeObject=%d", Kernel, NodeObject); |
| |
| /* Reference node object. */ |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Reference(Kernel, NodeObject)); |
| referenced = gcvTRUE; |
| |
| /* Allocated fd owns a reference. */ |
| gcmkONERROR(gckOS_Allocate( |
| Kernel->os, |
| gcmSIZEOF(gcsVIDMEM_NODE_FDPRIVATE), |
| (gctPOINTER *)&fdPrivate |
| )); |
| |
| fdPrivate->base.release = _ReleaseFdPrivate; |
| fdPrivate->kernel = Kernel; |
| fdPrivate->node = NodeObject; |
| |
| /* Allocated fd owns a reference. */ |
| gcmkONERROR(gckOS_GetFd("vidmem", &fdPrivate->base, Fd)); |
| |
| gcmkFOOTER_ARG("*Fd=%d", *Fd); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (referenced) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, NodeObject)); |
| } |
| |
| if (fdPrivate) |
| { |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, fdPrivate)); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_WrapUserMemory( |
| IN gckKERNEL Kernel, |
| IN gcsUSER_MEMORY_DESC_PTR Desc, |
| IN gceVIDMEM_TYPE Type, |
| OUT gckVIDMEM_NODE * NodeObject, |
| OUT gctUINT64 * Bytes |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gckVIDMEM_NODE nodeObject = gcvNULL; |
| gctBOOL found = gcvFALSE; |
| |
| gcmkHEADER_ARG("Kernel=0x%x", Kernel); |
| |
| gcmkVERIFY_ARGUMENT(Desc != gcvNULL); |
| |
| #if defined(CONFIG_DMA_SHARED_BUFFER) |
| if (Desc->flag & gcvALLOC_FLAG_DMABUF) |
| { |
| struct dma_buf *dmabuf; |
| int fd = (int)Desc->handle; |
| |
| if (fd >= 0) |
| { |
| /* Import dma buf handle. */ |
| dmabuf = dma_buf_get(fd); |
| |
| if (IS_ERR(dmabuf)) |
| return PTR_ERR(dmabuf); |
| |
| Desc->handle = -1; |
| Desc->dmabuf = gcmPTR_TO_UINT64(dmabuf); |
| |
| dma_buf_put(dmabuf); |
| } |
| else |
| { |
| if (!Desc->dmabuf) |
| { |
| gcmkPRINT("Wrap user memory: invalid dmabuf from user.\n"); |
| |
| gcmkFOOTER(); |
| return gcvSTATUS_INVALID_ARGUMENT; |
| } |
| |
| dmabuf = gcmUINT64_TO_PTR(Desc->dmabuf); |
| } |
| |
| if (dmabuf->ops == &_dmabuf_ops) |
| { |
| gctBOOL referenced = gcvFALSE; |
| nodeObject = dmabuf->priv; |
| |
| do |
| { |
| /* Reference the node. */ |
| gcmkERR_BREAK(gckVIDMEM_NODE_Reference(Kernel, nodeObject)); |
| referenced = gcvTRUE; |
| found = gcvTRUE; |
| |
| *NodeObject = nodeObject; |
| *Bytes = (gctUINT64)dmabuf->size; |
| } |
| while (gcvFALSE); |
| |
| if (gcmIS_ERROR(status) && referenced) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, nodeObject)); |
| } |
| } |
| } |
| #endif |
| |
| if (!found) |
| { |
| gckOS os = Kernel->os; |
| gcuVIDMEM_NODE_PTR node = gcvNULL; |
| |
| do |
| { |
| gctSIZE_T pageCountCpu = 0; |
| gctSIZE_T pageSizeCpu = 0; |
| gctPHYS_ADDR_T physicalAddress = 0; |
| |
| gcmkVERIFY_OK(gckOS_GetPageSize(os, &pageSizeCpu)); |
| |
| /* Allocate an gcuVIDMEM_NODE union. */ |
| gcmkERR_BREAK(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), (gctPOINTER*)&node)); |
| gckOS_ZeroMemory(node, gcmSIZEOF(gcuVIDMEM_NODE)); |
| |
| /* Initialize gcuVIDMEM_NODE union for virtual memory. */ |
| node->Virtual.kernel = Kernel; |
| |
| /* Wrap Memory. */ |
| gcmkERR_BREAK( |
| gckOS_WrapMemory(os, |
| Desc, |
| &node->Virtual.bytes, |
| &node->Virtual.physical, |
| &node->Virtual.contiguous, |
| &pageCountCpu)); |
| |
| /* Get base physical address. */ |
| gcmkERR_BREAK( |
| gckOS_GetPhysicalFromHandle(os, |
| node->Virtual.physical, |
| 0, |
| &physicalAddress)); |
| |
| /* Allocate handle for this video memory. */ |
| gcmkERR_BREAK(gckVIDMEM_NODE_Construct( |
| Kernel, |
| node, |
| Type, |
| gcvPOOL_VIRTUAL, |
| &nodeObject |
| )); |
| |
| node->Virtual.pageCount = (pageCountCpu * pageSizeCpu - |
| (physicalAddress & (pageSizeCpu - 1) & ~(4096 - 1))) >> 12; |
| |
| *NodeObject = nodeObject; |
| *Bytes = (gctUINT64)node->Virtual.bytes; |
| } |
| while (gcvFALSE); |
| |
| if (gcmIS_ERROR(status) && node) |
| { |
| /* Free the structure. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node)); |
| } |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_SetCommitStamp( |
| IN gckKERNEL Kernel, |
| IN gceENGINE Engine, |
| IN gckVIDMEM_NODE NodeObject, |
| IN gctUINT64 CommitStamp |
| ) |
| { |
| NodeObject->sync[Engine].commitStamp = CommitStamp; |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_GetCommitStamp( |
| IN gckKERNEL Kernel, |
| IN gceENGINE Engine, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gctUINT64_PTR CommitStamp |
| ) |
| { |
| *CommitStamp = NodeObject->sync[Engine].commitStamp; |
| return gcvSTATUS_OK; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckVIDMEM_NODE_Find |
| ** |
| ** Find gckVIDMEM_NODE object according to GPU address of specified core. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Kernel object, specifies core. |
| ** |
| ** gctUINT32 Address |
| ** GPU address to search. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckVIDMEM_NODE * NodeObject |
| ** Pointer to a variable hold found video memory node. |
| ** |
| ** gctUINT32 * Offset |
| ** The offset of specified GPU address in found video memory node. |
| */ |
| gceSTATUS |
| gckVIDMEM_NODE_Find( |
| IN gckKERNEL Kernel, |
| IN gctUINT32 Address, |
| OUT gckVIDMEM_NODE * NodeObject, |
| OUT gctUINT32 * Offset |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_NOT_FOUND; |
| gckVIDMEM_NODE nodeObject = gcvNULL; |
| gcuVIDMEM_NODE_PTR node = gcvNULL; |
| gckVIDMEM_BLOCK vidMemBlock = gcvNULL; |
| gcsLISTHEAD_PTR pos; |
| gceHARDWARE_TYPE hwType; |
| |
| gcmkVERIFY_OK( |
| gckKERNEL_GetHardwareType(Kernel, |
| &hwType)); |
| |
| gcmkVERIFY_OK( |
| gckOS_AcquireMutex(Kernel->os, |
| Kernel->db->videoMemListMutex, |
| gcvINFINITE)); |
| |
| gcmkLIST_FOR_EACH(pos, &Kernel->db->videoMemList) |
| { |
| nodeObject = (gckVIDMEM_NODE)gcmCONTAINEROF(pos, struct _gcsVIDMEM_NODE, link); |
| node = nodeObject->node; |
| vidMemBlock = node->VirtualChunk.parent; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| if (!node->VidMem.locked) |
| { |
| /* Don't check against unlocked node. */ |
| continue; |
| } |
| |
| if (Address >= node->VidMem.address && |
| Address <= node->VidMem.address + node->VidMem.bytes - 1) |
| { |
| *NodeObject = nodeObject; |
| |
| if (Offset) |
| { |
| *Offset = Address - node->VidMem.address; |
| } |
| |
| status = gcvSTATUS_OK; |
| break; |
| } |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| if (!node->VirtualChunk.lockeds[hwType]) |
| { |
| /* Don't check against unlocked node. */ |
| continue; |
| } |
| |
| if (Address >= node->VirtualChunk.addresses[hwType] && |
| Address <= node->VirtualChunk.addresses[hwType] + node->VirtualChunk.bytes - 1) |
| { |
| *NodeObject = nodeObject; |
| |
| if (Offset) |
| { |
| *Offset = Address - node->VirtualChunk.addresses[hwType]; |
| } |
| |
| status = gcvSTATUS_OK; |
| break; |
| } |
| } |
| else |
| { |
| if (!node->Virtual.lockeds[hwType]) |
| { |
| /* Don't check against unlocked node. */ |
| continue; |
| } |
| |
| if (Address >= node->Virtual.addresses[hwType] && |
| (Address <= node->Virtual.addresses[hwType] + node->Virtual.bytes - 1)) |
| { |
| *NodeObject = nodeObject; |
| |
| if (Offset) |
| { |
| *Offset = Address - node->Virtual.addresses[hwType]; |
| } |
| |
| status = gcvSTATUS_OK; |
| break; |
| } |
| } |
| } |
| |
| gcmkVERIFY_OK( |
| gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex)); |
| |
| return status; |
| } |
| |
| gceSTATUS |
| gckVIDMEM_NODE_IsContiguous( |
| IN gckKERNEL Kernel, |
| IN gckVIDMEM_NODE NodeObject, |
| OUT gctBOOL * Contiguous |
| ) |
| { |
| gceSTATUS status; |
| gckOS os = Kernel->os; |
| gctBOOL acquired = gcvFALSE; |
| gcuVIDMEM_NODE_PTR node = NodeObject->node; |
| gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; |
| |
| gcmkHEADER(); |
| |
| /* Grab the mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); |
| acquired = gcvTRUE; |
| |
| *Contiguous = gcvFALSE; |
| |
| if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) |
| { |
| *Contiguous = gcvTRUE; |
| } |
| else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) |
| { |
| if (vidMemBlock->contiguous) |
| { |
| *Contiguous = gcvTRUE; |
| } |
| } |
| else |
| { |
| if (node->Virtual.contiguous) |
| { |
| *Contiguous = gcvTRUE; |
| } |
| } |
| |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| /* Release the mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |