| /**************************************************************************** |
| * |
| * The MIT License (MIT) |
| * |
| * Copyright (c) 2014 - 2018 Vivante Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in |
| * all copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| * DEALINGS IN THE SOFTWARE. |
| * |
| ***************************************************************************** |
| * |
| * The GPL License (GPL) |
| * |
| * Copyright (C) 2014 - 2018 Vivante Corporation |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; either version 2 |
| * of the License, or (at your option) any later version. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| * |
| ***************************************************************************** |
| * |
| * Note: This software is released under dual MIT and GPL licenses. A |
| * recipient may use this file under the terms of either the MIT license or |
| * GPL License. If you wish to use only one license not the other, you can |
| * indicate your decision by deleting one of the above license notices in your |
| * version of this file. |
| * |
| *****************************************************************************/ |
| |
| |
| #include "gc_hal_kernel_precomp.h" |
| #include "gc_hal_kernel_context.h" |
| |
| #define _GC_OBJ_ZONE gcvZONE_COMMAND |
| |
| /******************************************************************************\ |
| ********************************* Support Code ********************************* |
| \******************************************************************************/ |
| |
| /******************************************************************************* |
| ** |
| ** _NewQueue |
| ** |
| ** Allocate a new command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctBOOL Stalled |
| ** Indicate if hardware is stalled already. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckCOMMAND Command |
| ** gckCOMMAND object has been updated with a new command queue. |
| */ |
| gceSTATUS |
| _NewQueue( |
| IN OUT gckCOMMAND Command, |
| IN gctBOOL Stalled |
| ) |
| { |
| gceSTATUS status; |
| gctINT currentIndex, newIndex; |
| gctPHYS_ADDR_T physical; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Switch to the next command buffer. */ |
| currentIndex = Command->index; |
| newIndex = (currentIndex + 1) % gcdCOMMAND_QUEUES; |
| |
| /* Wait for availability. */ |
| #if gcdDUMP_COMMAND |
| gcmkPRINT("@[kernel.waitsignal]"); |
| #endif |
| |
| gcmkONERROR(gckOS_WaitSignal( |
| Command->os, |
| Command->queues[newIndex].signal, |
| gcvFALSE, |
| gcvINFINITE |
| )); |
| |
| #if gcmIS_DEBUG(gcdDEBUG_TRACE) |
| if (newIndex < currentIndex) |
| { |
| Command->wrapCount += 1; |
| |
| gcmkTRACE_ZONE_N( |
| gcvLEVEL_INFO, gcvZONE_COMMAND, |
| 2 * 4, |
| "%s(%d): queue array wrapped around.\n", |
| __FUNCTION__, __LINE__ |
| ); |
| } |
| |
| gcmkTRACE_ZONE_N( |
| gcvLEVEL_INFO, gcvZONE_COMMAND, |
| 3 * 4, |
| "%s(%d): total queue wrap arounds %d.\n", |
| __FUNCTION__, __LINE__, Command->wrapCount |
| ); |
| |
| gcmkTRACE_ZONE_N( |
| gcvLEVEL_INFO, gcvZONE_COMMAND, |
| 3 * 4, |
| "%s(%d): switched to queue %d.\n", |
| __FUNCTION__, __LINE__, newIndex |
| ); |
| #endif |
| |
| /* Update gckCOMMAND object with new command queue. */ |
| Command->index = newIndex; |
| Command->newQueue = gcvTRUE; |
| #if USE_KERNEL_VIRTUAL_BUFFERS |
| if (Command->kernel->virtualCommandBuffer) |
| { |
| gckVIRTUAL_COMMAND_BUFFER_PTR commandBuffer = gcvNULL; |
| |
| Command->virtualMemory = Command->queues[newIndex].physical; |
| |
| commandBuffer = (gckVIRTUAL_COMMAND_BUFFER_PTR) Command->virtualMemory; |
| |
| Command->physHandle = commandBuffer->virtualBuffer.physical; |
| } |
| else |
| #endif |
| { |
| Command->physHandle = Command->queues[newIndex].physical; |
| } |
| |
| Command->logical = Command->queues[newIndex].logical; |
| Command->address = Command->queues[newIndex].address; |
| Command->offset = 0; |
| |
| gcmkONERROR(gckOS_GetPhysicalAddress( |
| Command->os, |
| Command->logical, |
| &physical |
| )); |
| |
| gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( |
| Command->os, |
| physical, |
| &physical |
| )); |
| |
| gcmkSAFECASTPHYSADDRT(Command->physical, physical); |
| |
| if (currentIndex != -1) |
| { |
| if (Stalled) |
| { |
| gckOS_Signal( |
| Command->os, |
| Command->queues[currentIndex].signal, |
| gcvTRUE |
| ); |
| } |
| else |
| { |
| /* Mark the command queue as available. */ |
| gcmkONERROR(gckEVENT_Signal( |
| Command->kernel->eventObj, |
| Command->queues[currentIndex].signal, |
| gcvKERNEL_COMMAND |
| )); |
| } |
| } |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("Command->index=%d", Command->index); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| _IncrementCommitAtom( |
| IN gckCOMMAND Command, |
| IN gctBOOL Increment |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctINT32 atomValue; |
| gctBOOL powerAcquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Extract the gckHARDWARE and gckEVENT objects. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Grab the power mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, hardware->powerMutex, gcvINFINITE |
| )); |
| powerAcquired = gcvTRUE; |
| |
| /* Increment the commit atom. */ |
| if (Increment) |
| { |
| gcmkONERROR(gckOS_AtomIncrement( |
| Command->os, Command->atomCommit, &atomValue |
| )); |
| } |
| else |
| { |
| gcmkONERROR(gckOS_AtomDecrement( |
| Command->os, Command->atomCommit, &atomValue |
| )); |
| } |
| |
| /* Release the power mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex( |
| Command->os, hardware->powerMutex |
| )); |
| powerAcquired = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (powerAcquired) |
| { |
| /* Release the power mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex( |
| Command->os, hardware->powerMutex |
| )); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| #if gcdSECURE_USER |
| gceSTATUS |
| _ProcessHints( |
| IN gckCOMMAND Command, |
| IN gctUINT32 ProcessID, |
| IN gcoCMDBUF CommandBuffer |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gckKERNEL kernel; |
| gctBOOL needCopy = gcvFALSE; |
| gcskSECURE_CACHE_PTR cache; |
| gctUINT8_PTR commandBufferLogical; |
| gctUINT8_PTR hintedData; |
| gctUINT32_PTR hintArray; |
| gctUINT i, hintCount; |
| |
| gcmkHEADER_ARG( |
| "Command=0x%08X ProcessID=%d CommandBuffer=0x%08X", |
| Command, ProcessID, CommandBuffer |
| ); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Reset state array pointer. */ |
| hintArray = gcvNULL; |
| |
| /* Get the kernel object. */ |
| kernel = Command->kernel; |
| |
| /* Get the cache form the database. */ |
| gcmkONERROR(gckKERNEL_GetProcessDBCache(kernel, ProcessID, &cache)); |
| |
| /* Determine the start of the command buffer. */ |
| commandBufferLogical |
| = (gctUINT8_PTR) CommandBuffer->logical |
| + CommandBuffer->startOffset; |
| |
| /* Determine the number of records in the state array. */ |
| hintCount = CommandBuffer->hintArrayTail - CommandBuffer->hintArray; |
| |
| /* Check wehther we need to copy the structures or not. */ |
| gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy)); |
| |
| /* Get access to the state array. */ |
| if (needCopy) |
| { |
| gctUINT copySize; |
| |
| if (Command->hintArrayAllocated && |
| (Command->hintArraySize < CommandBuffer->hintArraySize)) |
| { |
| gcmkONERROR(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray))); |
| Command->hintArraySize = gcvFALSE; |
| } |
| |
| if (!Command->hintArrayAllocated) |
| { |
| gctPOINTER pointer = gcvNULL; |
| |
| gcmkONERROR(gckOS_Allocate( |
| Command->os, |
| CommandBuffer->hintArraySize, |
| &pointer |
| )); |
| |
| Command->hintArray = gcmPTR_TO_UINT64(pointer); |
| Command->hintArrayAllocated = gcvTRUE; |
| Command->hintArraySize = CommandBuffer->hintArraySize; |
| } |
| |
| hintArray = gcmUINT64_TO_PTR(Command->hintArray); |
| copySize = hintCount * gcmSIZEOF(gctUINT32); |
| |
| gcmkONERROR(gckOS_CopyFromUserData( |
| Command->os, |
| hintArray, |
| gcmUINT64_TO_PTR(CommandBuffer->hintArray), |
| copySize |
| )); |
| } |
| else |
| { |
| gctPOINTER pointer = gcvNULL; |
| |
| gcmkONERROR(gckOS_MapUserPointer( |
| Command->os, |
| gcmUINT64_TO_PTR(CommandBuffer->hintArray), |
| CommandBuffer->hintArraySize, |
| &pointer |
| )); |
| |
| hintArray = pointer; |
| } |
| |
| /* Scan through the buffer. */ |
| for (i = 0; i < hintCount; i += 1) |
| { |
| /* Determine the location of the hinted data. */ |
| hintedData = commandBufferLogical + hintArray[i]; |
| |
| /* Map handle into physical address. */ |
| gcmkONERROR(gckKERNEL_MapLogicalToPhysical( |
| kernel, cache, (gctPOINTER) hintedData |
| )); |
| } |
| |
| OnError: |
| /* Get access to the state array. */ |
| if (!needCopy && (hintArray != gcvNULL)) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| gcmUINT64_TO_PTR(CommandBuffer->hintArray), |
| CommandBuffer->hintArraySize, |
| hintArray |
| )); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| #endif |
| |
| #if !gcdNULL_DRIVER |
| gceSTATUS |
| _FlushMMU( |
| IN gckCOMMAND Command |
| ) |
| { |
| #if gcdSECURITY |
| return gcvSTATUS_OK; |
| #else |
| gceSTATUS status; |
| gctUINT32 oldValue; |
| gckHARDWARE hardware = Command->kernel->hardware; |
| gctBOOL pause = gcvFALSE; |
| |
| gctUINT8_PTR pointer; |
| gctUINT32 address; |
| gctUINT32 eventBytes; |
| gctUINT32 endBytes; |
| gctUINT32 bufferSize; |
| gctUINT32 executeBytes; |
| gctUINT32 waitLinkBytes; |
| |
| gcmkONERROR(gckOS_AtomicExchange(Command->os, |
| hardware->pageTableDirty[gcvENGINE_RENDER], |
| 0, |
| &oldValue)); |
| |
| if (oldValue) |
| { |
| /* Page Table is upated, flush mmu before commit. */ |
| gcmkONERROR(gckHARDWARE_FlushMMU(hardware)); |
| |
| if ((oldValue & gcvPAGE_TABLE_DIRTY_BIT_FE) |
| && (!hardware->stallFEPrefetch) |
| ) |
| { |
| pause = gcvTRUE; |
| } |
| } |
| |
| if (pause) |
| { |
| /* Query size. */ |
| gcmkONERROR(gckHARDWARE_Event(hardware, gcvNULL, 0, gcvKERNEL_PIXEL, &eventBytes)); |
| gcmkONERROR(gckHARDWARE_End(hardware, gcvNULL, ~0U, &endBytes)); |
| |
| executeBytes = eventBytes + endBytes; |
| |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| hardware, |
| gcvNULL, |
| ~0U, |
| Command->offset + executeBytes, |
| &waitLinkBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| /* Reserve space. */ |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| executeBytes, |
| (gctPOINTER *)&pointer, |
| &bufferSize |
| )); |
| |
| /* Pointer to reserved address. */ |
| address = Command->address + Command->offset; |
| |
| /* Append EVENT(29). */ |
| gcmkONERROR(gckHARDWARE_Event( |
| hardware, |
| pointer, |
| 29, |
| gcvKERNEL_PIXEL, |
| &eventBytes |
| )); |
| |
| /* Append END. */ |
| pointer += eventBytes; |
| address += eventBytes; |
| |
| gcmkONERROR(gckHARDWARE_End(hardware, pointer, address, &endBytes)); |
| |
| #if USE_KERNEL_VIRTUAL_BUFFERS |
| if (hardware->kernel->virtualCommandBuffer) |
| { |
| gcmkONERROR(gckKERNEL_GetGPUAddress( |
| hardware->kernel, |
| pointer, |
| gcvFALSE, |
| Command->virtualMemory, |
| &hardware->lastEnd |
| )); |
| } |
| #endif |
| |
| gcmkONERROR(gckCOMMAND_Execute(Command, executeBytes)); |
| } |
| |
| return gcvSTATUS_OK; |
| OnError: |
| return status; |
| #endif |
| } |
| |
| gceSTATUS |
| _DummyDraw( |
| IN gckCOMMAND Command |
| ) |
| { |
| #if gcdSECURITY |
| return gcvSTATUS_OK; |
| #else |
| gceSTATUS status; |
| gckHARDWARE hardware = Command->kernel->hardware; |
| |
| gctUINT8_PTR pointer; |
| gctUINT32 bufferSize; |
| |
| gctUINT32 dummyDrawBytes; |
| gceDUMMY_DRAW_TYPE dummyDrawType = gcvDUMMY_DRAW_INVALID; |
| |
| if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_NEED_DUMMYDRAW)) |
| { |
| dummyDrawType = gcvDUMMY_DRAW_GC400; |
| } |
| |
| if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_DEFER_FILL_FIX) && |
| gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC)) |
| { |
| dummyDrawType = gcvDUMMY_DRAW_V60; |
| } |
| |
| if (dummyDrawType != gcvDUMMY_DRAW_INVALID) |
| { |
| gckHARDWARE_DummyDraw(hardware, gcvNULL, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); |
| |
| /* Reserve space. */ |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| dummyDrawBytes, |
| (gctPOINTER *)&pointer, |
| &bufferSize |
| )); |
| |
| gckHARDWARE_DummyDraw(hardware, pointer, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); |
| |
| gcmkONERROR(gckCOMMAND_Execute(Command, dummyDrawBytes)); |
| } |
| |
| return gcvSTATUS_OK; |
| OnError: |
| return status; |
| #endif |
| } |
| |
| #endif |
| |
| static void |
| _DumpBuffer( |
| IN gctPOINTER Buffer, |
| IN gctUINT32 GpuAddress, |
| IN gctSIZE_T Size |
| ) |
| { |
| gctSIZE_T i, line, left; |
| gctUINT32_PTR data = Buffer; |
| |
| line = Size / 32; |
| left = Size % 32; |
| |
| for (i = 0; i < line; i++) |
| { |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]); |
| data += 8; |
| GpuAddress += 8 * 4; |
| } |
| |
| switch(left) |
| { |
| case 28: |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6]); |
| break; |
| case 24: |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5]); |
| break; |
| case 20: |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4]); |
| break; |
| case 16: |
| gcmkPRINT("%08X : %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3]); |
| break; |
| case 12: |
| gcmkPRINT("%08X : %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2]); |
| break; |
| case 8: |
| gcmkPRINT("%08X : %08X %08X", |
| GpuAddress, data[0], data[1]); |
| break; |
| case 4: |
| gcmkPRINT("%08X : %08X", |
| GpuAddress, data[0]); |
| break; |
| default: |
| break; |
| } |
| } |
| |
| void |
| _DumpKernelCommandBuffer( |
| IN gckCOMMAND Command |
| ) |
| { |
| gctINT i; |
| gctUINT64 physical = 0; |
| gctUINT32 address; |
| gctPOINTER entry = gcvNULL; |
| |
| for (i = 0; i < gcdCOMMAND_QUEUES; i++) |
| { |
| entry = Command->queues[i].logical; |
| |
| gckOS_GetPhysicalAddress(Command->os, entry, &physical); |
| |
| gckOS_CPUPhysicalToGPUPhysical(Command->os, physical, &physical); |
| |
| gcmkPRINT("Kernel command buffer %d\n", i); |
| |
| gcmkSAFECASTPHYSADDRT(address, physical); |
| |
| _DumpBuffer(entry, address, Command->pageSize); |
| } |
| } |
| |
| #if !gcdNULL_DRIVER |
| gceSTATUS |
| _HandlePatchList( |
| IN gckCOMMAND Command, |
| IN gcoCMDBUF CommandBuffer, |
| IN gctBOOL NeedCopy, |
| OUT gctUINT64 *AsyncCommandStamp |
| ) |
| { |
| gceSTATUS status; |
| gcsPATCH_LIST * uList; |
| gcsPATCH_LIST * previous; |
| gcsPATCH_LIST * kList; |
| gctUINT64 asyncStamp = 0; |
| |
| gcmkHEADER_ARG( |
| "Command=0x%x CommandBuffer=0x%x NeedCopy=%d", |
| Command, CommandBuffer, NeedCopy |
| ); |
| |
| uList = gcmUINT64_TO_PTR(CommandBuffer->patchHead); |
| |
| while (uList) |
| { |
| gctUINT i; |
| |
| kList = gcvNULL; |
| previous = uList; |
| |
| gcmkONERROR(gckKERNEL_OpenUserData( |
| Command->kernel, |
| NeedCopy, |
| Command->kList, |
| uList, |
| gcmSIZEOF(gcsPATCH_LIST), |
| (gctPOINTER *)&kList |
| )); |
| |
| for (i = 0; i < kList->count; i++) |
| { |
| gctUINT64 stamp = 0; |
| gcsPATCH * patch = &kList->patch[i]; |
| |
| /* Touch video memory node. */ |
| gcmkVERIFY_OK(gckVIDMEM_SetCommitStamp(Command->kernel, gcvENGINE_RENDER, patch->handle, Command->commitStamp)); |
| |
| /* Get stamp touched async command buffer. */ |
| gcmkVERIFY_OK(gckVIDMEM_GetCommitStamp(Command->kernel, gcvENGINE_BLT, patch->handle, &stamp)); |
| |
| /* Find latest one. */ |
| asyncStamp = gcmMAX(asyncStamp, stamp); |
| } |
| |
| uList = kList->next; |
| |
| gcmkVERIFY_OK(gckKERNEL_CloseUserData( |
| Command->kernel, |
| NeedCopy, |
| gcvFALSE, |
| previous, |
| gcmSIZEOF(gcsPATCH_LIST), |
| (gctPOINTER *)&kList |
| )); |
| } |
| |
| if ((Command->asyncCommand != gcvNULL) |
| && (*(gctUINT64 *)Command->asyncCommand->fence->logical > asyncStamp) |
| ) |
| { |
| /* No need to wait for async command buffer. */ |
| *AsyncCommandStamp = 0; |
| } |
| else |
| { |
| /* Need to add a fence wait. */ |
| *AsyncCommandStamp = asyncStamp; |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (kList) |
| { |
| gcmkVERIFY_OK(gckKERNEL_CloseUserData( |
| Command->kernel, |
| NeedCopy, |
| gcvFALSE, |
| previous, |
| gcmSIZEOF(gcsPATCH_LIST), |
| (gctPOINTER *)&kList |
| )); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| _WaitForAsyncCommandStamp( |
| IN gckCOMMAND Command, |
| IN gctUINT64 Stamp |
| ) |
| { |
| gctUINT32 bytes; |
| gceSTATUS status; |
| gctUINT32 fenceAddress; |
| gctUINT32 bufferSize; |
| gctPOINTER pointer; |
| gcmkHEADER_ARG("Stamp = 0x%llx", Stamp); |
| |
| fenceAddress = Command->asyncCommand->fence->address; |
| |
| gcmkONERROR(gckHARDWARE_WaitFence(Command->kernel->hardware, |
| gcvNULL, |
| Stamp, |
| fenceAddress, |
| &bytes |
| )); |
| |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| bytes, |
| &pointer, |
| &bufferSize |
| )); |
| |
| gcmkONERROR(gckHARDWARE_WaitFence( |
| Command->kernel->hardware, |
| pointer, |
| Stamp, |
| fenceAddress, |
| &bytes |
| )); |
| |
| gcmkONERROR(gckCOMMAND_Execute(Command, bytes)); |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************\ |
| **************** Helper functions for parsing gcoCMDBUF ************************ |
| \******************************************************************************/ |
| static void |
| _GetCMDBUFSize( |
| IN gcoCMDBUF CommandBuffer, |
| OUT gctUINT_PTR CommandBufferSize |
| ) |
| { |
| *CommandBufferSize |
| = CommandBuffer->offset |
| + CommandBuffer->reservedTail |
| - CommandBuffer->startOffset; |
| } |
| |
| static void |
| _GetCMDBUFTail( |
| IN gcoCMDBUF CommandBuffer, |
| OUT gctUINT8_PTR * Tail |
| ) |
| { |
| gctUINT8_PTR commandBufferLogical; |
| gctUINT commandBufferSize; |
| |
| commandBufferLogical |
| = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) |
| + CommandBuffer->startOffset; |
| |
| _GetCMDBUFSize(CommandBuffer, &commandBufferSize); |
| |
| *Tail |
| = commandBufferLogical |
| + commandBufferSize |
| - CommandBuffer->reservedTail; |
| } |
| |
| static void |
| _ParseCMDBUFTail( |
| IN gckHARDWARE Hardware, |
| IN gcoCMDBUF CommandBuffer, |
| OUT gctUINT8_PTR * Fence, |
| OUT gctUINT8_PTR * Link |
| ) |
| { |
| gctUINT8_PTR tail; |
| |
| _GetCMDBUFTail(CommandBuffer, &tail); |
| |
| if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_FENCE_64BIT)) |
| { |
| *Fence = tail; |
| *Link = tail + gcdRENDER_FENCE_LENGTH; |
| } |
| else |
| { |
| *Fence = gcvNULL; |
| *Link = tail; |
| } |
| } |
| |
| gceSTATUS |
| _GetCMDBUFEntry( |
| IN gckCOMMAND Command, |
| IN gcoCMDBUF CommandBuffer, |
| OUT gctUINT32_PTR EntryAddress, |
| OUT gctUINT32_PTR EntryBytes |
| ) |
| { |
| gceSTATUS status; |
| gctUINT8_PTR commandBufferLogical; |
| gctUINT commandBufferSize; |
| gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer; |
| gctUINT32 commandBufferAddress; |
| gctUINT offset; |
| |
| commandBufferLogical |
| = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) |
| + CommandBuffer->startOffset; |
| |
| /* Get the hardware address. */ |
| if (Command->kernel->virtualCommandBuffer) |
| { |
| gckKERNEL kernel = Command->kernel; |
| |
| virtualCommandBuffer = gcmNAME_TO_PTR(CommandBuffer->physical); |
| |
| if (virtualCommandBuffer == gcvNULL) |
| { |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| } |
| |
| gcmkONERROR(gckKERNEL_GetGPUAddress( |
| Command->kernel, |
| commandBufferLogical, |
| gcvTRUE, |
| virtualCommandBuffer, |
| &commandBufferAddress |
| )); |
| } |
| else |
| { |
| gcmkONERROR(gckHARDWARE_ConvertLogical( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| gcvTRUE, |
| &commandBufferAddress |
| )); |
| } |
| |
| /* Get offset. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, gcvNULL, gcvPIPE_3D, &offset |
| )); |
| |
| _GetCMDBUFSize(CommandBuffer, &commandBufferSize); |
| |
| *EntryAddress = commandBufferAddress + offset; |
| *EntryBytes = commandBufferSize - offset; |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** Link a list of command buffer together to make them atomic. |
| ** Fence will be added in the last command buffer. |
| */ |
| gceSTATUS |
| _ProcessUserCommandBufferList( |
| IN gckCOMMAND Command, |
| IN gcoCMDBUF CommandBufferListHead, |
| OUT gcoCMDBUF * CommandBufferListTail |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL needCopy; |
| |
| gcoCMDBUF currentCMDBUF; |
| gcoCMDBUF currentCMDBUFUser = CommandBufferListHead; |
| |
| gckOS_QueryNeedCopy(Command->os, 0, &needCopy); |
| |
| /* Open first gcoCMDBUF object as currentCMDBUF. */ |
| gcmkONERROR(gckKERNEL_OpenUserData( |
| Command->kernel, |
| needCopy, |
| &Command->_commandBufferObject, |
| currentCMDBUFUser, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| (gctPOINTER *)¤tCMDBUF |
| )); |
| |
| /* Iterate the list. */ |
| while (currentCMDBUF->nextCMDBUF != 0) |
| { |
| gcoCMDBUF nextCMDBUFUser; |
| gcoCMDBUF nextCMDBUF; |
| gctUINT8_PTR fenceLogical = gcvNULL; |
| gctUINT8_PTR linkLogical; |
| gctUINT32 linkBytes = 8; |
| gctUINT32 linkLow; |
| gctUINT32 linkHigh; |
| |
| gctUINT32 entryAddress = 0; |
| gctUINT32 entryBytes = 0; |
| |
| nextCMDBUFUser |
| = gcmUINT64_TO_PTR(currentCMDBUF->nextCMDBUF); |
| |
| /* Open next gcoCMDBUF object as nextCMDBUF. */ |
| gcmkONERROR(gckKERNEL_OpenUserData( |
| Command->kernel, |
| needCopy, |
| &Command->_nextCMDBUF, |
| nextCMDBUFUser, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| (gctPOINTER *)&nextCMDBUF |
| )); |
| |
| /* Get the start hardware address of nextCMDBUF. */ |
| gcmkONERROR(_GetCMDBUFEntry(Command, |
| nextCMDBUF, |
| &entryAddress, |
| &entryBytes |
| )); |
| |
| /* Process current gcoCMDBUF object. */ |
| _ParseCMDBUFTail( |
| Command->kernel->hardware, |
| currentCMDBUF, |
| &fenceLogical, |
| &linkLogical |
| ); |
| |
| /* Don't send fence in the middle of gcoCMDBUF list. */ |
| if (fenceLogical != gcvNULL) |
| { |
| gctUINT i = gcdRENDER_FENCE_LENGTH / gcmSIZEOF(gctUINT32) / 2; |
| |
| /* Fill NOPs in space reserved for fence. */ |
| while (i--) |
| { |
| gctSIZE_T nopBytes = 8; |
| gcmkONERROR(gckHARDWARE_Nop(Command->kernel->hardware, fenceLogical, &nopBytes)); |
| fenceLogical += nopBytes; |
| } |
| } |
| |
| /* Generate a LINK from the end of current command buffer |
| ** to the start of next command buffer. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| Command->kernel->hardware, |
| linkLogical, |
| entryAddress, |
| entryBytes, |
| &linkBytes, |
| &linkLow, |
| &linkHigh |
| )); |
| |
| /* Close current gcoCMDBUF object which is processed. */ |
| gcmkVERIFY_OK(gckKERNEL_CloseUserData( |
| Command->kernel, |
| needCopy, |
| gcvFALSE, |
| currentCMDBUFUser, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| (gctPOINTER *)¤tCMDBUF |
| )); |
| |
| /* Advance to next gcoCMDBUF object. */ |
| currentCMDBUFUser = nextCMDBUFUser; |
| currentCMDBUF = nextCMDBUF; |
| } |
| |
| gcmkVERIFY_OK(gckKERNEL_CloseUserData( |
| Command->kernel, |
| needCopy, |
| gcvFALSE, |
| currentCMDBUFUser, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| (gctPOINTER *)¤tCMDBUF |
| )); |
| |
| /* Return the tail of the list. */ |
| *CommandBufferListTail = currentCMDBUFUser; |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| return status; |
| } |
| #endif |
| |
| /******************************************************************************\ |
| ****************************** gckCOMMAND API Code ****************************** |
| \******************************************************************************/ |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Construct |
| ** |
| ** Construct a new gckCOMMAND object. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckCOMMAND * Command |
| ** Pointer to a variable that will hold the pointer to the gckCOMMAND |
| ** object. |
| */ |
| gceSTATUS |
| gckCOMMAND_Construct( |
| IN gckKERNEL Kernel, |
| OUT gckCOMMAND * Command |
| ) |
| { |
| gckOS os; |
| gckCOMMAND command = gcvNULL; |
| gceSTATUS status; |
| gctINT i; |
| gctPOINTER pointer = gcvNULL; |
| gctSIZE_T pageSize; |
| |
| gcmkHEADER_ARG("Kernel=0x%x", Kernel); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); |
| gcmkVERIFY_ARGUMENT(Command != gcvNULL); |
| |
| /* Extract the gckOS object. */ |
| os = Kernel->os; |
| |
| /* Allocate the gckCOMMAND structure. */ |
| gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckCOMMAND), &pointer)); |
| command = pointer; |
| |
| /* Reset the entire object. */ |
| gcmkONERROR(gckOS_ZeroMemory(command, gcmSIZEOF(struct _gckCOMMAND))); |
| |
| /* Initialize the gckCOMMAND object.*/ |
| command->object.type = gcvOBJ_COMMAND; |
| command->kernel = Kernel; |
| command->os = os; |
| |
| /* Get the command buffer requirements. */ |
| gcmkONERROR(gckHARDWARE_QueryCommandBuffer( |
| Kernel->hardware, |
| gcvENGINE_RENDER, |
| &command->alignment, |
| &command->reservedHead, |
| gcvNULL |
| )); |
| |
| /* Create the command queue mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &command->mutexQueue)); |
| |
| /* Create the context switching mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContext)); |
| |
| /* Create the context switching mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContextSeq)); |
| |
| /* Create the power management semaphore. */ |
| gcmkONERROR(gckOS_CreateSemaphore(os, &command->powerSemaphore)); |
| |
| /* Create the commit atom. */ |
| gcmkONERROR(gckOS_AtomConstruct(os, &command->atomCommit)); |
| |
| /* Get the page size from teh OS. */ |
| gcmkONERROR(gckOS_GetPageSize(os, &pageSize)); |
| |
| gcmkSAFECASTSIZET(command->pageSize, pageSize); |
| |
| /* Get process ID. */ |
| gcmkONERROR(gckOS_GetProcessID(&command->kernelProcessID)); |
| |
| /* Set hardware to pipe 0. */ |
| command->pipeSelect = gcvPIPE_INVALID; |
| |
| /* Pre-allocate the command queues. */ |
| for (i = 0; i < gcdCOMMAND_QUEUES; ++i) |
| { |
| #if USE_KERNEL_VIRTUAL_BUFFERS |
| if (Kernel->virtualCommandBuffer) |
| { |
| gcmkONERROR(gckKERNEL_AllocateVirtualCommandBuffer( |
| Kernel, |
| gcvFALSE, |
| &pageSize, |
| &command->queues[i].physical, |
| &command->queues[i].logical |
| )); |
| |
| gcmkONERROR(gckKERNEL_GetGPUAddress( |
| Kernel, |
| command->queues[i].logical, |
| gcvFALSE, |
| command->queues[i].physical, |
| &command->queues[i].address |
| )); |
| } |
| else |
| #endif |
| { |
| gctUINT32 allocFlag; |
| |
| #if gcdENABLE_CACHEABLE_COMMAND_BUFFER |
| allocFlag = gcvALLOC_FLAG_CACHEABLE | gcvALLOC_FLAG_CONTIGUOUS; |
| #else |
| allocFlag = gcvALLOC_FLAG_CONTIGUOUS; |
| #endif |
| |
| gcmkONERROR(gckOS_AllocateNonPagedMemory( |
| os, |
| gcvFALSE, |
| allocFlag, |
| &pageSize, |
| &command->queues[i].physical, |
| &command->queues[i].logical |
| )); |
| |
| gcmkONERROR(gckHARDWARE_ConvertLogical( |
| Kernel->hardware, |
| command->queues[i].logical, |
| gcvFALSE, |
| &command->queues[i].address |
| )); |
| |
| gcmkONERROR(gckMMU_FillFlatMapping( |
| Kernel->mmu, command->queues[i].address, pageSize |
| )); |
| } |
| |
| gcmkONERROR(gckOS_CreateSignal( |
| os, gcvFALSE, &command->queues[i].signal |
| )); |
| |
| gcmkONERROR(gckOS_Signal( |
| os, command->queues[i].signal, gcvTRUE |
| )); |
| } |
| |
| #if gcdRECORD_COMMAND |
| gcmkONERROR(gckRECORDER_Construct(os, Kernel->hardware, &command->recorder)); |
| #endif |
| |
| gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsPATCH_LIST), &command->kList)); |
| |
| gcmkONERROR(gckFENCE_Create( |
| os, Kernel, &command->fence |
| )); |
| |
| /* No command queue in use yet. */ |
| command->index = -1; |
| command->logical = gcvNULL; |
| command->newQueue = gcvFALSE; |
| |
| /* Command is not yet running. */ |
| command->running = gcvFALSE; |
| |
| /* Command queue is idle. */ |
| command->idle = gcvTRUE; |
| |
| /* Commit stamp start from 1. */ |
| command->commitStamp = 1; |
| |
| /* END event signal not created. */ |
| command->endEventSignal = gcvNULL; |
| |
| command->dummyDraw = gcvTRUE; |
| |
| /* Return pointer to the gckCOMMAND object. */ |
| *Command = command; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Command=0x%x", *Command); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Roll back. */ |
| if (command != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckCOMMAND_Destroy(command)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Destroy |
| ** |
| ** Destroy an gckCOMMAND object. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to destroy. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Destroy( |
| IN gckCOMMAND Command |
| ) |
| { |
| gctINT i; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Stop the command queue. */ |
| gcmkVERIFY_OK(gckCOMMAND_Stop(Command)); |
| |
| for (i = 0; i < gcdCOMMAND_QUEUES; ++i) |
| { |
| if (Command->queues[i].signal) |
| { |
| gcmkVERIFY_OK(gckOS_DestroySignal( |
| Command->os, Command->queues[i].signal |
| )); |
| } |
| |
| if (Command->queues[i].logical) |
| { |
| #if USE_KERNEL_VIRTUAL_BUFFERS |
| if (Command->kernel->virtualCommandBuffer) |
| { |
| gcmkVERIFY_OK(gckKERNEL_DestroyVirtualCommandBuffer( |
| Command->kernel, |
| Command->pageSize, |
| Command->queues[i].physical, |
| Command->queues[i].logical |
| )); |
| } |
| else |
| #endif |
| { |
| gcmkVERIFY_OK(gckOS_FreeNonPagedMemory( |
| Command->os, |
| Command->pageSize, |
| Command->queues[i].physical, |
| Command->queues[i].logical |
| )); |
| } |
| } |
| } |
| |
| /* END event signal. */ |
| if (Command->endEventSignal != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckOS_DestroySignal( |
| Command->os, Command->endEventSignal |
| )); |
| } |
| |
| if (Command->mutexContext) |
| { |
| /* Delete the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContext)); |
| } |
| |
| if (Command->mutexContextSeq != gcvNULL) |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContextSeq)); |
| |
| if (Command->mutexQueue) |
| { |
| /* Delete the command queue mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexQueue)); |
| } |
| |
| if (Command->powerSemaphore) |
| { |
| /* Destroy the power management semaphore. */ |
| gcmkVERIFY_OK(gckOS_DestroySemaphore(Command->os, Command->powerSemaphore)); |
| } |
| |
| if (Command->atomCommit) |
| { |
| /* Destroy the commit atom. */ |
| gcmkVERIFY_OK(gckOS_AtomDestroy(Command->os, Command->atomCommit)); |
| } |
| |
| #if gcdSECURE_USER |
| /* Free state array. */ |
| if (Command->hintArrayAllocated) |
| { |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray))); |
| Command->hintArrayAllocated = gcvFALSE; |
| } |
| #endif |
| |
| #if gcdRECORD_COMMAND |
| gckRECORDER_Destory(Command->os, Command->recorder); |
| #endif |
| |
| if (Command->stateMap) |
| { |
| gcmkOS_SAFE_FREE(Command->os, Command->stateMap); |
| } |
| |
| if (Command->kList) |
| { |
| gcmkOS_SAFE_FREE(Command->os, Command->kList); |
| } |
| |
| if (Command->fence) |
| { |
| gcmkVERIFY_OK(gckFENCE_Destory(Command->os, Command->fence)); |
| } |
| |
| /* Mark object as unknown. */ |
| Command->object.type = gcvOBJ_UNKNOWN; |
| |
| /* Free the gckCOMMAND object. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, Command)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_EnterCommit |
| ** |
| ** Acquire command queue synchronization objects. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to destroy. |
| ** |
| ** gctBOOL FromPower |
| ** Determines whether the call originates from inside the power |
| ** management or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_EnterCommit( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctBOOL atomIncremented = gcvFALSE; |
| gctBOOL semaAcquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Extract the gckHARDWARE and gckEVENT objects. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| if (!FromPower) |
| { |
| /* Increment COMMIT atom to let power management know that a commit is |
| ** in progress. */ |
| gcmkONERROR(_IncrementCommitAtom(Command, gcvTRUE)); |
| atomIncremented = gcvTRUE; |
| |
| /* Notify the system the GPU has a commit. */ |
| gcmkONERROR(gckOS_Broadcast(Command->os, |
| hardware, |
| gcvBROADCAST_GPU_COMMIT)); |
| |
| /* Acquire the power management semaphore. */ |
| gcmkONERROR(gckOS_AcquireSemaphore(Command->os, |
| Command->powerSemaphore)); |
| semaAcquired = gcvTRUE; |
| } |
| |
| /* Grab the conmmand queue mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(Command->os, |
| Command->mutexQueue, |
| gcvINFINITE)); |
| |
| /* Success. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (semaAcquired) |
| { |
| /* Release the power management semaphore. */ |
| gcmkVERIFY_OK(gckOS_ReleaseSemaphore( |
| Command->os, Command->powerSemaphore |
| )); |
| } |
| |
| if (atomIncremented) |
| { |
| /* Decrement the commit atom. */ |
| gcmkVERIFY_OK(_IncrementCommitAtom( |
| Command, gcvFALSE |
| )); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_ExitCommit |
| ** |
| ** Release command queue synchronization objects. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to destroy. |
| ** |
| ** gctBOOL FromPower |
| ** Determines whether the call originates from inside the power |
| ** management or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_ExitCommit( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower |
| ) |
| { |
| gceSTATUS status; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Release the power mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); |
| |
| if (!FromPower) |
| { |
| /* Release the power management semaphore. */ |
| gcmkONERROR(gckOS_ReleaseSemaphore(Command->os, |
| Command->powerSemaphore)); |
| |
| /* Decrement the commit atom. */ |
| gcmkONERROR(_IncrementCommitAtom(Command, gcvFALSE)); |
| } |
| |
| /* Success. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Start |
| ** |
| ** Start up the command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to start. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Start( |
| IN gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctUINT32 waitOffset = 0; |
| gctUINT32 waitLinkBytes; |
| gctPOINTER logical; |
| gctUINT32 physical; |
| gctUINT32 address; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| if (Command->running) |
| { |
| /* Command queue already running. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /* Extract the gckHARDWARE object. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Query the size of WAIT/LINK command sequence. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| hardware, |
| gcvNULL, |
| ~0U, |
| Command->offset, |
| &waitLinkBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| if ((Command->pageSize - Command->offset < waitLinkBytes) |
| || (Command->logical == gcvNULL) |
| ) |
| { |
| /* Start at beginning of a new queue. */ |
| gcmkONERROR(_NewQueue(Command, gcvTRUE)); |
| } |
| |
| logical = (gctUINT8_PTR) Command->logical + Command->offset; |
| physical = Command->physical + Command->offset; |
| address = Command->address + Command->offset; |
| |
| /* Append WAIT/LINK. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| hardware, |
| logical, |
| address, |
| 0, |
| &waitLinkBytes, |
| &waitOffset, |
| &Command->waitSize |
| )); |
| |
| Command->waitLogical = (gctUINT8_PTR) logical + waitOffset; |
| Command->waitPhysical = physical + waitOffset; |
| Command->waitAddress = address + waitOffset; |
| Command->waitOffset = waitOffset; |
| |
| /* Flush the cache for the wait/link. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| 0, |
| Command->physHandle, |
| Command->offset, |
| logical, |
| waitLinkBytes |
| )); |
| |
| /* Adjust offset. */ |
| Command->offset += waitLinkBytes; |
| Command->newQueue = gcvFALSE; |
| |
| #if gcdSECURITY |
| /* Start FE by calling security service. */ |
| gckKERNEL_SecurityStartCommand( |
| Command->kernel |
| ); |
| #else |
| /* Enable command processor. */ |
| gcmkONERROR(gckHARDWARE_Execute( |
| hardware, |
| address, |
| waitLinkBytes |
| )); |
| #endif |
| |
| /* Command queue is running. */ |
| Command->running = gcvTRUE; |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Stop |
| ** |
| ** Stop the command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to stop. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Stop( |
| IN gckCOMMAND Command |
| ) |
| { |
| gckHARDWARE hardware; |
| gceSTATUS status; |
| gctUINT32 idle; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| if (!Command->running) |
| { |
| /* Command queue is not running. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /* Extract the gckHARDWARE object. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| if (gckHARDWARE_IsFeatureAvailable(hardware, |
| gcvFEATURE_END_EVENT) == gcvSTATUS_TRUE) |
| { |
| /* Allocate the signal. */ |
| if (Command->endEventSignal == gcvNULL) |
| { |
| gcmkONERROR(gckOS_CreateSignal(Command->os, |
| gcvTRUE, |
| &Command->endEventSignal)); |
| } |
| |
| /* Append the END EVENT command to trigger the signal. */ |
| gcmkONERROR(gckEVENT_Stop(Command->kernel->eventObj, |
| Command->kernelProcessID, |
| Command->physHandle, |
| Command->offset + Command->waitOffset, |
| Command->waitLogical, |
| Command->waitAddress, |
| Command->endEventSignal, |
| &Command->waitSize)); |
| } |
| else |
| { |
| /* Replace last WAIT with END. */ |
| gcmkONERROR(gckHARDWARE_End( |
| hardware, |
| Command->waitLogical, |
| Command->waitAddress, |
| &Command->waitSize |
| )); |
| |
| #if USE_KERNEL_VIRTUAL_BUFFERS |
| if (hardware->kernel->virtualCommandBuffer) |
| { |
| gcmkONERROR(gckKERNEL_GetGPUAddress( |
| hardware->kernel, |
| Command->waitLogical, |
| gcvFALSE, |
| Command->virtualMemory, |
| &hardware->lastEnd |
| )); |
| } |
| #endif |
| |
| #if gcdSECURITY |
| gcmkONERROR(gckKERNEL_SecurityExecute( |
| Command->kernel, Command->waitLogical, 8 |
| )); |
| #endif |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail(Command->kernel->hardware, |
| Command->logical, |
| Command->offset)); |
| |
| /* Flush the cache for the END. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| 0, |
| Command->physHandle, |
| Command->offset + Command->waitOffset, |
| Command->waitLogical, |
| Command->waitSize |
| )); |
| |
| /* Wait for idle. */ |
| gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle)); |
| } |
| |
| /* Command queue is no longer running. */ |
| Command->running = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Commit |
| ** |
| ** Commit a command buffer to the command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** gckCONTEXT Context |
| ** Pointer to a gckCONTEXT object. |
| ** |
| ** gcoCMDBUF CommandBuffer |
| ** Pointer to a gcoCMDBUF object. |
| ** |
| ** gcsSTATE_DELTA_PTR StateDelta |
| ** Pointer to the state delta. |
| ** |
| ** gctUINT32 ProcessID |
| ** Current process ID. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Commit( |
| IN gckCOMMAND Command, |
| IN gckCONTEXT Context, |
| IN gcoCMDBUF CommandBuffer, |
| IN gcsSTATE_DELTA_PTR StateDelta, |
| IN gctUINT32 ProcessID, |
| IN gctBOOL Shared, |
| IN gctUINT32 Index, |
| OUT gctUINT64_PTR CommitStamp, |
| OUT gctBOOL_PTR ContextSwitched |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL commitEntered = gcvFALSE; |
| gctBOOL contextAcquired = gcvFALSE; |
| gckHARDWARE hardware; |
| gctBOOL needCopy = gcvFALSE; |
| gctBOOL commandBufferMapped = gcvFALSE; |
| gcoCMDBUF commandBufferObject = gcvNULL; |
| gctBOOL stall = gcvFALSE; |
| gctBOOL contextSwitched = gcvFALSE; |
| |
| #if !gcdNULL_DRIVER |
| gcsCONTEXT_PTR contextBuffer; |
| gctPHYS_ADDR_T commandBufferPhysical; |
| gctUINT8_PTR commandBufferLogical = gcvNULL; |
| gctUINT32 commandBufferAddress = 0; |
| gctUINT8_PTR commandBufferTail = gcvNULL; |
| gctUINT commandBufferSize; |
| gctSIZE_T nopBytes; |
| gctUINT32 pipeBytes; |
| gctUINT32 linkBytes; |
| gctSIZE_T bytes; |
| gctUINT32 offset; |
| gctPOINTER entryLogical; |
| gctUINT32 entryAddress; |
| gctUINT32 entryBytes; |
| gctUINT32 exitOffset; |
| gctPOINTER exitLogical; |
| gctUINT32 exitAddress; |
| gctUINT32 exitBytes; |
| gctUINT32 waitLinkPhysical; |
| gctPOINTER waitLinkLogical; |
| gctUINT32 waitLinkAddress; |
| gctUINT32 waitLinkBytes; |
| gctUINT32 waitOffset; |
| gctUINT32 waitSize; |
| |
| #ifdef __QNXNTO__ |
| gctPOINTER userCommandBufferLogical = gcvNULL; |
| gctBOOL userCommandBufferLogicalMapped = gcvFALSE; |
| #endif |
| |
| #if gcdPROCESS_ADDRESS_SPACE |
| gckMMU mmu; |
| gctUINT32 oldValue; |
| #endif |
| |
| #if gcdDUMP_COMMAND |
| gctPOINTER contextDumpLogical = gcvNULL; |
| gctSIZE_T contextDumpBytes = 0; |
| gctPOINTER bufferDumpLogical = gcvNULL; |
| gctSIZE_T bufferDumpBytes = 0; |
| # endif |
| gctUINT32 exitLinkLow = 0, exitLinkHigh = 0; |
| gctUINT32 entryLinkLow = 0, entryLinkHigh = 0; |
| gctUINT32 commandLinkLow = 0, commandLinkHigh = 0; |
| |
| gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer = gcvNULL; |
| gctUINT64 asyncCommandStamp = 0; |
| gcoCMDBUF lastCommandBuffer = gcvNULL; |
| gctPOINTER pointer = gcvNULL; |
| gckKERNEL kernel = Command->kernel; |
| |
| gctPHYS_ADDR contextPhysHandle = gcvNULL; |
| gctPHYS_ADDR physHandle = gcvNULL; |
| #endif |
| |
| gcmkHEADER_ARG( |
| "Command=0x%x CommandBuffer=0x%x ProcessID=%d", |
| Command, CommandBuffer, ProcessID |
| ); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| #if gcdPROCESS_ADDRESS_SPACE |
| gcmkONERROR(gckKERNEL_GetProcessMMU(Command->kernel, &mmu)); |
| |
| gcmkONERROR(gckOS_AtomicExchange(Command->os, |
| mmu->pageTableDirty[Command->kernel->core], |
| 0, |
| &oldValue)); |
| #else |
| #endif |
| |
| /* Acquire the command queue. */ |
| gcmkONERROR(gckCOMMAND_EnterCommit(Command, gcvFALSE)); |
| commitEntered = gcvTRUE; |
| |
| #if !gcdNULL_DRIVER |
| gcmkONERROR(_ProcessUserCommandBufferList( |
| Command, |
| CommandBuffer, |
| &lastCommandBuffer |
| )); |
| #endif |
| |
| /* Acquire the context switching mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContext, gcvINFINITE |
| )); |
| contextAcquired = gcvTRUE; |
| |
| /* Extract the gckHARDWARE and gckEVENT objects. */ |
| hardware = Command->kernel->hardware; |
| |
| /* Check wehther we need to copy the structures or not. */ |
| gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy)); |
| |
| #if gcdNULL_DRIVER |
| /* Context switch required? */ |
| if ((Context != gcvNULL) && (Command->currContext != Context)) |
| { |
| /* Yes, merge in the deltas. */ |
| gckCONTEXT_Update(Context, ProcessID, StateDelta); |
| |
| /* Update the current context. */ |
| Command->currContext = Context; |
| |
| contextSwitched = gcvTRUE; |
| } |
| #else |
| if (needCopy) |
| { |
| commandBufferObject = &Command->_commandBufferObject; |
| |
| gcmkONERROR(gckOS_CopyFromUserData( |
| Command->os, |
| commandBufferObject, |
| CommandBuffer, |
| gcmSIZEOF(struct _gcoCMDBUF) |
| )); |
| |
| gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER); |
| } |
| else |
| { |
| gcmkONERROR(gckOS_MapUserPointer( |
| Command->os, |
| CommandBuffer, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| &pointer |
| )); |
| |
| commandBufferObject = pointer; |
| |
| gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER); |
| commandBufferMapped = gcvTRUE; |
| } |
| |
| gcmkONERROR(_HandlePatchList(Command, commandBufferObject, needCopy, &asyncCommandStamp)); |
| |
| /* Query the size of NOP command. */ |
| gcmkONERROR(gckHARDWARE_Nop( |
| hardware, gcvNULL, &nopBytes |
| )); |
| |
| /* Query the size of pipe select command sequence. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| hardware, gcvNULL, gcvPIPE_3D, &pipeBytes |
| )); |
| |
| /* Query the size of LINK command. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL |
| )); |
| |
| /* Compute the command buffer entry and the size. */ |
| commandBufferLogical |
| = (gctUINT8_PTR) gcmUINT64_TO_PTR(commandBufferObject->logical) |
| + commandBufferObject->startOffset; |
| |
| |
| /* Get the hardware address. */ |
| if (Command->kernel->virtualCommandBuffer) |
| { |
| virtualCommandBuffer = gcmNAME_TO_PTR(commandBufferObject->physical); |
| physHandle = virtualCommandBuffer->virtualBuffer.physical; |
| |
| if (virtualCommandBuffer == gcvNULL) |
| { |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| } |
| |
| gcmkONERROR(gckKERNEL_GetGPUAddress( |
| Command->kernel, |
| commandBufferLogical, |
| gcvTRUE, |
| virtualCommandBuffer, |
| &commandBufferAddress |
| )); |
| } |
| else |
| { |
| physHandle = gcmNAME_TO_PTR(commandBufferObject->physical); |
| |
| gcmkONERROR(gckHARDWARE_ConvertLogical( |
| hardware, |
| commandBufferLogical, |
| gcvTRUE, |
| &commandBufferAddress |
| )); |
| } |
| |
| #ifdef __QNXNTO__ |
| userCommandBufferLogical = (gctPOINTER) commandBufferLogical; |
| |
| gcmkONERROR(gckOS_MapUserPointer( |
| Command->os, |
| userCommandBufferLogical, |
| 0, |
| &pointer)); |
| |
| commandBufferLogical = pointer; |
| |
| userCommandBufferLogicalMapped = gcvTRUE; |
| |
| gcmkONERROR(gckOS_GetPhysicalAddress( |
| Command->os, |
| commandBufferLogical, |
| &commandBufferPhysical |
| )); |
| |
| #else |
| /* Get the physical address. */ |
| gcmkONERROR(gckOS_UserLogicalToPhysical( |
| Command->os, |
| commandBufferLogical, |
| &commandBufferPhysical |
| )); |
| #endif |
| |
| commandBufferSize |
| = commandBufferObject->offset |
| + commandBufferObject->reservedTail |
| - commandBufferObject->startOffset; |
| |
| gcmkONERROR(_FlushMMU(Command)); |
| |
| if (Command->dummyDraw == gcvTRUE && |
| Context != gcvNULL) |
| { |
| Command->dummyDraw = gcvFALSE; |
| gcmkONERROR(_DummyDraw(Command)); |
| } |
| |
| if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT) && asyncCommandStamp != 0) |
| { |
| gcmkONERROR(_WaitForAsyncCommandStamp(Command, asyncCommandStamp)); |
| } |
| |
| /* Get the current offset. */ |
| offset = Command->offset; |
| |
| /* Compute number of bytes left in current kernel command queue. */ |
| bytes = Command->pageSize - offset; |
| |
| /* Query the size of WAIT/LINK command sequence. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| hardware, |
| gcvNULL, |
| ~0U, |
| offset, |
| &waitLinkBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| /* Is there enough space in the current command queue? */ |
| if (bytes < waitLinkBytes) |
| { |
| /* No, create a new one. */ |
| gcmkONERROR(_NewQueue(Command, gcvFALSE)); |
| |
| /* Get the new current offset. */ |
| offset = Command->offset; |
| |
| /* Recompute the number of bytes in the new kernel command queue. */ |
| bytes = Command->pageSize - offset; |
| gcmkASSERT(bytes >= waitLinkBytes); |
| } |
| |
| /* Compute the location if WAIT/LINK command sequence. */ |
| waitLinkPhysical = Command->physical + offset; |
| waitLinkLogical = (gctUINT8_PTR) Command->logical + offset; |
| waitLinkAddress = Command->address + offset; |
| |
| /* Context switch required? */ |
| if (Context == gcvNULL) |
| { |
| /* See if we have to switch pipes for the command buffer. */ |
| if (commandBufferObject->entryPipe == Command->pipeSelect) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the entry command buffer pipes |
| ** are different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Compute the entry. */ |
| entryLogical = commandBufferLogical + offset; |
| entryAddress = commandBufferAddress + offset; |
| entryBytes = commandBufferSize - offset; |
| |
| Command->currContext = gcvNULL; |
| } |
| #if gcdDEBUG_OPTION && gcdDEBUG_FORCE_CONTEXT_UPDATE |
| else if (1) |
| #else |
| else if (Command->currContext != Context) |
| #endif |
| { |
| /* Get the current context buffer. */ |
| contextBuffer = Context->buffer; |
| |
| /* Yes, merge in the deltas. */ |
| gcmkONERROR(gckCONTEXT_Update(Context, ProcessID, StateDelta)); |
| |
| contextSwitched = gcvTRUE; |
| |
| /*************************************************************** |
| ** SWITCHING CONTEXT. |
| */ |
| |
| /* Determine context buffer entry offset. */ |
| offset = (Command->pipeSelect == gcvPIPE_3D) |
| |
| /* Skip pipe switching sequence. */ |
| ? Context->entryOffset3D + Context->pipeSelectBytes |
| |
| /* Do not skip pipe switching sequence. */ |
| : Context->entryOffset3D; |
| |
| /* Compute the entry. */ |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset; |
| entryAddress = contextBuffer->address + offset; |
| entryBytes = Context->bufferSize - offset; |
| |
| /* See if we have to switch pipes between the context |
| and command buffers. */ |
| if (commandBufferObject->entryPipe == gcvPIPE_3D) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the initial context pipes are |
| different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Generate a LINK from the context buffer to |
| the command buffer. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| contextBuffer->link3D, |
| commandBufferAddress + offset, |
| commandBufferSize - offset, |
| &linkBytes, |
| &commandLinkLow, |
| &commandLinkHigh |
| )); |
| |
| #if USE_KERNEL_VIRTUAL_BUFFERS |
| if (Command->kernel->virtualCommandBuffer) |
| { |
| gckVIRTUAL_COMMAND_BUFFER_PTR commandBuffer = (gckVIRTUAL_COMMAND_BUFFER_PTR) contextBuffer->physical; |
| |
| contextPhysHandle = commandBuffer->virtualBuffer.physical; |
| } |
| else |
| #endif |
| { |
| contextPhysHandle = contextBuffer->physical; |
| } |
| |
| /* Flush the context buffer cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| 0, |
| contextPhysHandle, |
| entryAddress - contextBuffer->address, |
| entryLogical, |
| entryBytes |
| )); |
| |
| /* Update the current context. */ |
| Command->currContext = Context; |
| |
| #if gcdDUMP_COMMAND |
| contextDumpLogical = entryLogical; |
| contextDumpBytes = entryBytes; |
| #endif |
| |
| #if gcdSECURITY |
| /* Commit context buffer to trust zone. */ |
| gckKERNEL_SecurityExecute( |
| Command->kernel, |
| entryLogical, |
| entryBytes - 8 |
| ); |
| #endif |
| |
| #if gcdRECORD_COMMAND |
| gckRECORDER_Record( |
| Command->recorder, |
| gcvNULL, |
| 0xFFFFFFFF, |
| entryLogical, |
| entryBytes |
| ); |
| #endif |
| } |
| |
| /* Same context. */ |
| else |
| { |
| /* See if we have to switch pipes for the command buffer. */ |
| if (commandBufferObject->entryPipe == Command->pipeSelect) |
| { |
| /* Skip pipe switching sequence. */ |
| offset = pipeBytes; |
| } |
| else |
| { |
| /* The current hardware and the entry command buffer pipes |
| ** are different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| commandBufferObject->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| } |
| |
| /* Compute the entry. */ |
| entryLogical = commandBufferLogical + offset; |
| entryAddress = commandBufferAddress + offset; |
| entryBytes = commandBufferSize - offset; |
| } |
| |
| #if gcdDUMP_COMMAND |
| bufferDumpLogical = commandBufferLogical + offset; |
| bufferDumpBytes = commandBufferSize - offset; |
| #endif |
| |
| #if gcdSECURE_USER |
| /* Process user hints. */ |
| gcmkONERROR(_ProcessHints(Command, ProcessID, commandBufferObject)); |
| #endif |
| |
| /* Determine the location to jump to for the command buffer being |
| ** scheduled. */ |
| if (Command->newQueue) |
| { |
| /* New command queue, jump to the beginning of it. */ |
| exitOffset = 0; |
| exitLogical = Command->logical; |
| exitAddress = Command->address; |
| exitBytes = Command->offset + waitLinkBytes; |
| } |
| else |
| { |
| /* Still within the preexisting command queue, jump to the new |
| WAIT/LINK command sequence. */ |
| exitOffset = offset; |
| exitLogical = waitLinkLogical; |
| exitAddress = waitLinkAddress; |
| exitBytes = waitLinkBytes; |
| } |
| |
| /* Add a new WAIT/LINK command sequence. When the command buffer which is |
| currently being scheduled is fully executed by the GPU, the FE will |
| jump to this WAIT/LINK sequence. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| hardware, |
| waitLinkLogical, |
| waitLinkAddress, |
| offset, |
| &waitLinkBytes, |
| &waitOffset, |
| &waitSize |
| )); |
| |
| /* Flush the command queue cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| 0, |
| Command->physHandle, |
| exitOffset, |
| exitLogical, |
| exitBytes |
| )); |
| |
| /* Determine the location of the TAIL in the command buffer. */ |
| commandBufferTail |
| = commandBufferLogical |
| + commandBufferSize |
| - commandBufferObject->reservedTail; |
| |
| /* Generate command which writes out commit stamp. */ |
| if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT)) |
| { |
| gctUINT32 bytes; |
| |
| gcmkONERROR(gckHARDWARE_Fence( |
| hardware, |
| gcvENGINE_RENDER, |
| commandBufferTail, |
| Command->fence->address, |
| Command->commitStamp, |
| &bytes |
| )); |
| |
| commandBufferTail += gcdRENDER_FENCE_LENGTH; |
| } |
| |
| /* Generate a LINK from the end of the command buffer being scheduled |
| back to the kernel command queue. */ |
| #if !gcdSECURITY |
| if (Shared == gcvFALSE) |
| { |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| commandBufferTail, |
| exitAddress, |
| exitBytes, |
| &linkBytes, |
| &exitLinkLow, |
| &exitLinkHigh |
| )); |
| } |
| else |
| { |
| gctUINT8_PTR link = commandBufferTail + Index * 16; |
| gctSIZE_T bytes = 8; |
| |
| gcmkONERROR(gckHARDWARE_ChipEnable( |
| hardware, |
| link, |
| (gceCORE_3D_MASK)(1 << hardware->kernel->chipID), |
| &bytes |
| )); |
| |
| link += bytes; |
| |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| link, |
| exitAddress, |
| exitBytes, |
| &linkBytes, |
| &exitLinkLow, |
| &exitLinkHigh |
| )); |
| |
| link += linkBytes; |
| } |
| #endif |
| |
| /* Flush the command buffer cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| ProcessID, |
| physHandle, |
| commandBufferObject->startOffset, |
| commandBufferLogical, |
| commandBufferSize |
| )); |
| |
| #if gcdRECORD_COMMAND |
| gckRECORDER_Record( |
| Command->recorder, |
| commandBufferLogical + offset, |
| commandBufferSize - offset, |
| gcvNULL, |
| 0xFFFFFFFF |
| ); |
| |
| gckRECORDER_AdvanceIndex(Command->recorder, Command->commitStamp); |
| #endif |
| |
| #if gcdSECURITY |
| /* Submit command buffer to trust zone. */ |
| gckKERNEL_SecurityExecute( |
| Command->kernel, |
| commandBufferLogical + offset, |
| commandBufferSize - offset - 8 |
| ); |
| #else |
| /* Generate a LINK from the previous WAIT/LINK command sequence to the |
| entry determined above (either the context or the command buffer). |
| This LINK replaces the WAIT instruction from the previous WAIT/LINK |
| pair, therefore we use WAIT metrics for generation of this LINK. |
| This action will execute the entire sequence. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| hardware, |
| Command->waitLogical, |
| entryAddress, |
| entryBytes, |
| &Command->waitSize, |
| &entryLinkLow, |
| &entryLinkHigh |
| )); |
| #endif |
| |
| #if gcdLINK_QUEUE_SIZE |
| if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_USER_COMMAND) |
| { |
| gcuQUEUEDATA data; |
| |
| gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); |
| |
| data.linkData.start = entryAddress; |
| data.linkData.end = entryAddress + entryBytes; |
| data.linkData.linkLow = entryLinkLow; |
| data.linkData.linkHigh = entryLinkHigh; |
| |
| gckQUEUE_Enqueue(&hardware->linkQueue, &data); |
| |
| if (commandBufferAddress + offset != entryAddress) |
| { |
| data.linkData.start = commandBufferAddress + offset; |
| data.linkData.end = commandBufferAddress + commandBufferSize; |
| data.linkData.linkLow = commandLinkLow; |
| data.linkData.linkHigh = commandLinkHigh; |
| |
| gckQUEUE_Enqueue(&hardware->linkQueue, &data); |
| } |
| |
| if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) |
| { |
| data.linkData.start = exitAddress; |
| data.linkData.end = exitAddress + exitBytes; |
| data.linkData.linkLow = exitLinkLow; |
| data.linkData.linkHigh = exitLinkHigh; |
| |
| /* Dump kernel command.*/ |
| gckQUEUE_Enqueue(&hardware->linkQueue, &data); |
| } |
| } |
| #endif |
| |
| /* Flush the cache for the link. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| 0, |
| Command->physHandle, |
| Command->offset + waitOffset, |
| Command->waitLogical, |
| Command->waitSize |
| )); |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| Command->waitLogical, |
| Command->waitSize, |
| gcvDUMP_BUFFER_LINK, |
| gcvFALSE |
| ); |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| contextDumpLogical, |
| contextDumpBytes, |
| gcvDUMP_BUFFER_CONTEXT, |
| gcvFALSE |
| ); |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| bufferDumpLogical, |
| bufferDumpBytes, |
| gcvDUMP_BUFFER_USER, |
| gcvFALSE |
| ); |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| waitLinkLogical, |
| waitLinkBytes, |
| gcvDUMP_BUFFER_WAITLINK, |
| gcvFALSE |
| ); |
| |
| /* Update the current pipe. */ |
| Command->pipeSelect = commandBufferObject->exitPipe; |
| |
| /* Update command queue offset. */ |
| Command->offset += waitLinkBytes; |
| Command->newQueue = gcvFALSE; |
| |
| /* Update address of last WAIT. */ |
| Command->waitPhysical = waitLinkPhysical + waitOffset; |
| Command->waitLogical = (gctUINT8_PTR)waitLinkLogical + waitOffset; |
| Command->waitAddress = waitLinkAddress + waitOffset; |
| Command->waitSize = waitSize; |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail( |
| hardware, Command->logical, Command->offset |
| )); |
| |
| #if gcdDUMP_COMMAND |
| gcmkPRINT("@[kernel.commit]"); |
| #endif |
| #endif /* gcdNULL_DRIVER */ |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| contextAcquired = gcvFALSE; |
| |
| *CommitStamp = Command->commitStamp; |
| *ContextSwitched = contextSwitched; |
| |
| Command->commitStamp++; |
| |
| stall = gcvFALSE; |
| |
| #if gcdLINK_QUEUE_SIZE |
| if (Command->kernel->stuckDump == gcvSTUCK_DUMP_STALL_COMMAND) |
| { |
| if ((Command->commitStamp % (gcdLINK_QUEUE_SIZE/2)) == 0) |
| { |
| /* If only context buffer and command buffer is recorded, |
| ** each commit costs 2 slot in queue, to make sure command |
| ** causing stuck is recorded, number of pending command buffer |
| ** is limited to (gckLINK_QUEUE_SIZE/2) |
| */ |
| stall = gcvTRUE; |
| } |
| } |
| #endif |
| |
| /* Release the command queue. */ |
| gcmkONERROR(gckCOMMAND_ExitCommit(Command, gcvFALSE)); |
| commitEntered = gcvFALSE; |
| |
| if (status == gcvSTATUS_INTERRUPTED) |
| { |
| gcmkTRACE( |
| gcvLEVEL_INFO, |
| "%s(%d): Intterupted in gckEVENT_Submit", |
| __FUNCTION__, __LINE__ |
| ); |
| status = gcvSTATUS_OK; |
| } |
| else |
| { |
| gcmkONERROR(status); |
| } |
| |
| #ifdef __QNXNTO__ |
| if (userCommandBufferLogicalMapped) |
| { |
| gcmkONERROR(gckOS_UnmapUserPointer( |
| Command->os, |
| userCommandBufferLogical, |
| 0, |
| commandBufferLogical)); |
| |
| userCommandBufferLogicalMapped = gcvFALSE; |
| } |
| #endif |
| |
| /* Unmap the command buffer pointer. */ |
| if (commandBufferMapped) |
| { |
| gcmkONERROR(gckOS_UnmapUserPointer( |
| Command->os, |
| CommandBuffer, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| commandBufferObject |
| )); |
| |
| commandBufferMapped = gcvFALSE; |
| } |
| |
| /* Return status. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (contextAcquired) |
| { |
| /* Release the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| } |
| |
| if (commitEntered) |
| { |
| /* Release the command queue mutex. */ |
| gcmkVERIFY_OK(gckCOMMAND_ExitCommit(Command, gcvFALSE)); |
| } |
| |
| #ifdef __QNXNTO__ |
| if (userCommandBufferLogicalMapped) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| userCommandBufferLogical, |
| 0, |
| commandBufferLogical)); |
| } |
| #endif |
| |
| /* Unmap the command buffer pointer. */ |
| if (commandBufferMapped) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| CommandBuffer, |
| gcmSIZEOF(struct _gcoCMDBUF), |
| commandBufferObject |
| )); |
| } |
| |
| /* Return status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Reserve |
| ** |
| ** Reserve space in the command queue. Also acquire the command queue mutex. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctSIZE_T RequestedBytes |
| ** Number of bytes previously reserved. |
| ** |
| ** OUTPUT: |
| ** |
| ** gctPOINTER * Buffer |
| ** Pointer to a variable that will receive the address of the reserved |
| ** space. |
| ** |
| ** gctSIZE_T * BufferSize |
| ** Pointer to a variable that will receive the number of bytes |
| ** available in the command queue. |
| */ |
| gceSTATUS |
| gckCOMMAND_Reserve( |
| IN gckCOMMAND Command, |
| IN gctUINT32 RequestedBytes, |
| OUT gctPOINTER * Buffer, |
| OUT gctUINT32 * BufferSize |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 bytes; |
| gctUINT32 requiredBytes; |
| gctUINT32 requestedAligned; |
| |
| gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Compute aligned number of reuested bytes. */ |
| requestedAligned = gcmALIGN(RequestedBytes, Command->alignment); |
| |
| /* Another WAIT/LINK command sequence will have to be appended after |
| the requested area being reserved. Compute the number of bytes |
| required for WAIT/LINK at the location after the reserved area. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| Command->kernel->hardware, |
| gcvNULL, |
| ~0U, |
| Command->offset + requestedAligned, |
| &requiredBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| /* Compute total number of bytes required. */ |
| requiredBytes += requestedAligned; |
| |
| /* Compute number of bytes available in command queue. */ |
| bytes = Command->pageSize - Command->offset; |
| |
| /* Is there enough space in the current command queue? */ |
| if (bytes < requiredBytes) |
| { |
| /* Create a new command queue. */ |
| gcmkONERROR(_NewQueue(Command, gcvFALSE)); |
| |
| /* Recompute the number of bytes in the new kernel command queue. */ |
| bytes = Command->pageSize - Command->offset; |
| |
| /* Still not enough space? */ |
| if (bytes < requiredBytes) |
| { |
| /* Rare case, not enough room in command queue. */ |
| gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); |
| } |
| } |
| |
| /* Return pointer to empty slot command queue. */ |
| *Buffer = (gctUINT8 *) Command->logical + Command->offset; |
| |
| /* Return number of bytes left in command queue. */ |
| *BufferSize = bytes; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Buffer=0x%x *BufferSize=%lu", *Buffer, *BufferSize); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Execute |
| ** |
| ** Execute a previously reserved command queue by appending a WAIT/LINK command |
| ** sequence after it and modifying the last WAIT into a LINK command. The |
| ** command FIFO mutex will be released whether this function succeeds or not. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctSIZE_T RequestedBytes |
| ** Number of bytes previously reserved. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Execute( |
| IN gckCOMMAND Command, |
| IN gctUINT32 RequestedBytes |
| ) |
| { |
| gceSTATUS status; |
| |
| gctUINT32 waitLinkPhysical; |
| gctUINT8_PTR waitLinkLogical; |
| gctUINT32 waitLinkAddress; |
| gctUINT32 waitLinkOffset; |
| gctUINT32 waitLinkBytes; |
| |
| gctUINT32 waitPhysical; |
| gctPOINTER waitLogical; |
| gctUINT32 waitAddress; |
| gctUINT32 waitOffset; |
| gctUINT32 waitBytes; |
| |
| gctUINT32 linkLow, linkHigh; |
| |
| gctUINT32 execOffset; |
| gctPOINTER execLogical; |
| gctUINT32 execAddress; |
| gctUINT32 execBytes; |
| |
| gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Compute offset for WAIT/LINK. */ |
| waitLinkOffset = Command->offset + RequestedBytes; |
| |
| /* Compute number of bytes left in command queue. */ |
| waitLinkBytes = Command->pageSize - waitLinkOffset; |
| |
| /* Compute the location if WAIT/LINK command sequence. */ |
| waitLinkPhysical = Command->physical + waitLinkOffset; |
| waitLinkLogical = (gctUINT8_PTR) Command->logical + waitLinkOffset; |
| waitLinkAddress = Command->address + waitLinkOffset; |
| |
| /* Append WAIT/LINK in command queue. */ |
| gcmkONERROR(gckHARDWARE_WaitLink( |
| Command->kernel->hardware, |
| waitLinkLogical, |
| waitLinkAddress, |
| waitLinkOffset, |
| &waitLinkBytes, |
| &waitOffset, |
| &waitBytes |
| )); |
| |
| /* Compute the location if WAIT command. */ |
| waitPhysical = waitLinkPhysical + waitOffset; |
| waitLogical = waitLinkLogical + waitOffset; |
| waitAddress = waitLinkAddress + waitOffset; |
| |
| /* Determine the location to jump to for the command buffer being |
| ** scheduled. */ |
| if (Command->newQueue) |
| { |
| /* New command queue, jump to the beginning of it. */ |
| execOffset = 0; |
| execLogical = Command->logical; |
| execAddress = Command->address; |
| execBytes = waitLinkOffset + waitLinkBytes; |
| } |
| else |
| { |
| /* Still within the preexisting command queue, jump directly to the |
| reserved area. */ |
| execOffset = Command->offset; |
| execLogical = (gctUINT8 *) Command->logical + Command->offset; |
| execAddress = Command->address + Command->offset; |
| execBytes = RequestedBytes + waitLinkBytes; |
| } |
| |
| /* Flush the cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| 0, |
| Command->physHandle, |
| execOffset, |
| execLogical, |
| execBytes |
| )); |
| |
| /* Convert the last WAIT into a LINK. */ |
| gcmkONERROR(gckHARDWARE_Link( |
| Command->kernel->hardware, |
| Command->waitLogical, |
| execAddress, |
| execBytes, |
| &Command->waitSize, |
| &linkLow, |
| &linkHigh |
| )); |
| |
| /* Flush the cache. */ |
| gcmkONERROR(gckOS_CacheClean( |
| Command->os, |
| 0, |
| Command->physHandle, |
| waitLinkOffset + waitOffset, |
| Command->waitLogical, |
| Command->waitSize |
| )); |
| |
| #if gcdLINK_QUEUE_SIZE |
| if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) |
| { |
| gcuQUEUEDATA data; |
| |
| gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); |
| |
| data.linkData.start = execAddress; |
| data.linkData.end = execAddress + execBytes; |
| data.linkData.linkLow = linkLow; |
| data.linkData.linkHigh = linkHigh; |
| |
| gckQUEUE_Enqueue(&Command->kernel->hardware->linkQueue, &data); |
| } |
| #endif |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| Command->waitLogical, |
| Command->waitSize, |
| gcvDUMP_BUFFER_LINK, |
| gcvFALSE |
| ); |
| |
| gcmkDUMPCOMMAND( |
| Command->os, |
| execLogical, |
| execBytes, |
| gcvDUMP_BUFFER_KERNEL, |
| gcvFALSE |
| ); |
| |
| /* Update the pointer to the last WAIT. */ |
| Command->waitPhysical = waitPhysical; |
| Command->waitLogical = waitLogical; |
| Command->waitAddress = waitAddress; |
| Command->waitSize = waitBytes; |
| |
| /* Update the command queue. */ |
| Command->offset += RequestedBytes + waitLinkBytes; |
| Command->newQueue = gcvFALSE; |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail( |
| Command->kernel->hardware, Command->logical, Command->offset |
| )); |
| |
| #if gcdDUMP_COMMAND |
| gcmkPRINT("@[kernel.execute]"); |
| #endif |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Stall |
| ** |
| ** The calling thread will be suspended until the command queue has been |
| ** completed. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctBOOL FromPower |
| ** Determines whether the call originates from inside the power |
| ** management or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Stall( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower |
| ) |
| { |
| #if gcdNULL_DRIVER |
| /* Do nothing with infinite hardware. */ |
| return gcvSTATUS_OK; |
| #else |
| gckOS os; |
| gckHARDWARE hardware; |
| gckEVENT eventObject; |
| gceSTATUS status; |
| gctSIGNAL signal = gcvNULL; |
| gctUINT timer = 0; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Extract the gckOS object pointer. */ |
| os = Command->os; |
| gcmkVERIFY_OBJECT(os, gcvOBJ_OS); |
| |
| /* Extract the gckHARDWARE object pointer. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Extract the gckEVENT object pointer. */ |
| eventObject = Command->kernel->eventObj; |
| gcmkVERIFY_OBJECT(eventObject, gcvOBJ_EVENT); |
| |
| /* Allocate the signal. */ |
| gcmkONERROR(gckOS_CreateSignal(os, gcvTRUE, &signal)); |
| |
| /* Append the EVENT command to trigger the signal. */ |
| gcmkONERROR(gckEVENT_Signal(eventObject, signal, gcvKERNEL_PIXEL)); |
| |
| /* Submit the event queue. */ |
| gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower)); |
| |
| #if gcdDUMP_COMMAND |
| gcmkPRINT("@[kernel.stall]"); |
| #endif |
| |
| if (status == gcvSTATUS_CHIP_NOT_READY) |
| { |
| /* Error. */ |
| goto OnError; |
| } |
| |
| do |
| { |
| /* Wait for the signal. */ |
| status = gckOS_WaitSignal(os, signal, !FromPower, gcdGPU_ADVANCETIMER); |
| |
| if (status == gcvSTATUS_TIMEOUT) |
| { |
| #if gcmIS_DEBUG(gcdDEBUG_CODE) |
| gctUINT32 idle; |
| |
| /* Read idle register. */ |
| gcmkVERIFY_OK(gckHARDWARE_GetIdle( |
| hardware, gcvFALSE, &idle |
| )); |
| |
| gcmkTRACE( |
| gcvLEVEL_ERROR, |
| "%s(%d): idle=%08x", |
| __FUNCTION__, __LINE__, idle |
| ); |
| |
| gcmkVERIFY_OK(gckOS_MemoryBarrier(os, gcvNULL)); |
| #endif |
| |
| /* Advance timer. */ |
| timer += gcdGPU_ADVANCETIMER; |
| } |
| else if (status == gcvSTATUS_INTERRUPTED) |
| { |
| gcmkONERROR(gcvSTATUS_INTERRUPTED); |
| } |
| |
| } |
| while (gcmIS_ERROR(status)); |
| |
| /* Bail out on timeout. */ |
| if (gcmIS_ERROR(status)) |
| { |
| /* Broadcast the stuck GPU. */ |
| gcmkONERROR(gckOS_Broadcast( |
| os, hardware, gcvBROADCAST_GPU_STUCK |
| )); |
| } |
| |
| /* Delete the signal. */ |
| gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (signal != gcvNULL) |
| { |
| /* Free the signal. */ |
| gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| #endif |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Attach |
| ** |
| ** Attach user process. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** gctUINT32 ProcessID |
| ** Current process ID. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckCONTEXT * Context |
| ** Pointer to a variable that will receive a pointer to a new |
| ** gckCONTEXT object. |
| ** |
| ** gctSIZE_T * StateCount |
| ** Pointer to a variable that will receive the number of states |
| ** in the context buffer. |
| */ |
| #if (gcdENABLE_3D || gcdENABLE_2D) |
| gceSTATUS |
| gckCOMMAND_Attach( |
| IN gckCOMMAND Command, |
| OUT gckCONTEXT * Context, |
| OUT gctSIZE_T * MaxState, |
| OUT gctUINT32 * NumStates, |
| IN gctUINT32 ProcessID |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=0x%x", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Acquire the context switching mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContext, gcvINFINITE |
| )); |
| acquired = gcvTRUE; |
| |
| /* Construct a gckCONTEXT object. */ |
| gcmkONERROR(gckCONTEXT_Construct( |
| Command->os, |
| Command->kernel->hardware, |
| ProcessID, |
| Context |
| )); |
| |
| /* Return the number of states in the context. */ |
| * MaxState = (* Context)->maxState; |
| * NumStates = (* Context)->numStates; |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Context=0x%x", *Context); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Release mutex. */ |
| if (acquired) |
| { |
| /* Release the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| #endif |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Detach |
| ** |
| ** Detach user process. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** gckCONTEXT Context |
| ** Pointer to a gckCONTEXT object to be destroyed. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Detach( |
| IN gckCOMMAND Command, |
| IN gckCONTEXT Context |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=0x%x Context=0x%x", Command, Context); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Acquire the context switching mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContext, gcvINFINITE |
| )); |
| acquired = gcvTRUE; |
| |
| /* Construct a gckCONTEXT object. */ |
| gcmkONERROR(gckCONTEXT_Destroy(Context)); |
| |
| if (Command->currContext == Context) |
| { |
| /* Detach from gckCOMMAND object if the destoryed context is current context. */ |
| Command->currContext = gcvNULL; |
| } |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Release mutex. */ |
| if (acquired) |
| { |
| /* Release the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_DumpExecutingBuffer |
| ** |
| ** Dump the command buffer which GPU is executing. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_DumpExecutingBuffer( |
| IN gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| gckVIRTUAL_COMMAND_BUFFER_PTR buffer = gcvNULL; |
| gctUINT32 gpuAddress; |
| gctSIZE_T pageCount; |
| gctPOINTER entry = gcvNULL; |
| gckOS os = Command->os; |
| gckKERNEL kernel = Command->kernel; |
| gctUINT32 i; |
| gctUINT32 dumpRear; |
| gckQUEUE queue = &kernel->hardware->linkQueue; |
| gctSIZE_T bytes; |
| gckLINKDATA linkData; |
| gcuQUEUEDATA * queueData; |
| gctUINT32 offset; |
| gctPOINTER entryDump; |
| gctUINT32 pid; |
| gctUINT8 processName[24] = {0}; |
| gctPHYS_ADDR_T cpuPhysical; |
| |
| gcmkPRINT("**************************\n"); |
| gcmkPRINT("**** COMMAND BUF DUMP ****\n"); |
| gcmkPRINT("**************************\n"); |
| |
| gcmkPRINT(" Submitted commit stamp = %lld", Command->commitStamp - 1); |
| gcmkPRINT(" Executed commit stamp = %lld", *(gctUINT64_PTR)Command->fence->logical); |
| |
| gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); |
| gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); |
| |
| gcmkPRINT("DMA Address 0x%08X, memory around:", gpuAddress); |
| |
| /* Search and dump memory around DMA address. */ |
| if (kernel->virtualCommandBuffer) |
| { |
| status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer); |
| } |
| else |
| { |
| status = gcvSTATUS_OK; |
| } |
| |
| if (gcmIS_SUCCESS(status)) |
| { |
| if (kernel->virtualCommandBuffer) |
| { |
| gcmkVERIFY_OK(gckOS_CreateKernelVirtualMapping( |
| os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, &entry, &pageCount)); |
| |
| offset = gpuAddress - buffer->virtualBuffer.gpuAddress; |
| |
| entryDump = entry; |
| |
| /* Dump one pages. */ |
| bytes = 4096; |
| |
| /* Align to page. */ |
| offset &= 0xfffff000; |
| |
| /* Kernel address of page where stall point stay. */ |
| entryDump = (gctUINT8_PTR)entryDump + offset; |
| |
| /* Align to page. */ |
| gpuAddress &= 0xfffff000; |
| } |
| else |
| { |
| gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical)); |
| |
| gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, 4096, &entry)); |
| |
| /* Align to page start. */ |
| entryDump = (gctPOINTER)((gctUINTPTR_T)entry & ~0xFFF); |
| gpuAddress = gpuAddress & ~0xFFF; |
| bytes = 4096; |
| } |
| |
| gcmkPRINT("User Command Buffer:\n"); |
| _DumpBuffer(entryDump, gpuAddress, bytes); |
| |
| if (kernel->virtualCommandBuffer) |
| { |
| gcmkVERIFY_OK(gckOS_DestroyKernelVirtualMapping( |
| os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, entry)); |
| } |
| else |
| { |
| gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, 4096)); |
| } |
| } |
| else |
| { |
| _DumpKernelCommandBuffer(Command); |
| } |
| |
| /* Dump link queue. */ |
| if (queue->count) |
| { |
| gcmkPRINT("Dump Level is %d, dump %d valid record in link queue:", |
| Command->kernel->stuckDump, queue->count); |
| |
| dumpRear = queue->count; |
| |
| for (i = 0; i < dumpRear; i++) |
| { |
| gckQUEUE_GetData(queue, i, &queueData); |
| |
| linkData = &queueData->linkData; |
| |
| /* Get gpu address of this command buffer. */ |
| gpuAddress = linkData->start; |
| bytes = linkData->end - gpuAddress; |
| |
| pid = linkData->pid; |
| |
| gckOS_GetProcessNameByPid(pid, 16, processName); |
| |
| if (kernel->virtualCommandBuffer) |
| { |
| buffer = gcvNULL; |
| |
| /* Get the whole buffer. */ |
| status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer); |
| |
| if (gcmIS_ERROR(status)) |
| { |
| /* Get kernel address of kernel command buffer. */ |
| status = gckCOMMAND_AddressInKernelCommandBuffer( |
| kernel->command, gpuAddress, &entry); |
| |
| if (gcmIS_ERROR(status)) |
| { |
| status = gckHARDWARE_AddressInHardwareFuncions( |
| kernel->hardware, gpuAddress, &entry); |
| |
| if (gcmIS_ERROR(status)) |
| { |
| gcmkPRINT("Buffer [%08X - %08X] not found, may be freed", |
| linkData->start, |
| linkData->end); |
| continue; |
| } |
| } |
| |
| offset = 0; |
| gcmkPRINT("Kernel Command Buffer: %08X, %08X", linkData->linkLow, linkData->linkHigh); |
| } |
| else |
| { |
| /* Get kernel logical for dump. */ |
| if (buffer->virtualBuffer.kernelLogical) |
| { |
| /* Get kernel logical directly if it is a context buffer. */ |
| entry = buffer->virtualBuffer.kernelLogical; |
| gcmkPRINT("Context Buffer: %08X, %08X PID:%d %s", |
| linkData->linkLow, linkData->linkHigh, linkData->pid, processName); |
| } |
| else |
| { |
| /* Make it accessiable by kernel if it is a user command buffer. */ |
| gcmkVERIFY_OK( |
| gckOS_CreateKernelVirtualMapping(os, |
| buffer->virtualBuffer.physical, |
| buffer->virtualBuffer.bytes, |
| &entry, |
| &pageCount)); |
| gcmkPRINT("User Command Buffer: %08X, %08X PID:%d %s", |
| linkData->linkLow, linkData->linkHigh, linkData->pid, processName); |
| } |
| |
| offset = gpuAddress - buffer->virtualBuffer.gpuAddress; |
| } |
| |
| /* Dump from the entry. */ |
| _DumpBuffer((gctUINT8_PTR)entry + offset, gpuAddress, bytes); |
| |
| /* Release kernel logical address if neccessary. */ |
| if (buffer && !buffer->virtualBuffer.kernelLogical) |
| { |
| gcmkVERIFY_OK( |
| gckOS_DestroyKernelVirtualMapping(os, |
| buffer->virtualBuffer.physical, |
| buffer->virtualBuffer.bytes, |
| entry)); |
| } |
| } |
| else |
| { |
| gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical)); |
| |
| gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, bytes, &entry)); |
| |
| gcmkPRINT("Command Buffer: %08X, %08X PID:%d %s", |
| linkData->linkLow, linkData->linkHigh, linkData->pid, processName); |
| |
| _DumpBuffer((gctUINT8_PTR)entry, gpuAddress, bytes); |
| |
| gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, bytes)); |
| } |
| } |
| } |
| |
| return gcvSTATUS_OK; |
| } |
| |
| gceSTATUS |
| gckCOMMAND_AddressInKernelCommandBuffer( |
| IN gckCOMMAND Command, |
| IN gctUINT32 Address, |
| OUT gctPOINTER * Pointer |
| ) |
| { |
| gctINT i; |
| |
| for (i = 0; i < gcdCOMMAND_QUEUES; i++) |
| { |
| if ((Address >= Command->queues[i].address) |
| && (Address < (Command->queues[i].address + Command->pageSize)) |
| ) |
| { |
| *Pointer = (gctUINT8_PTR)Command->queues[i].logical |
| + (Address - Command->queues[i].address) |
| ; |
| |
| return gcvSTATUS_OK; |
| } |
| } |
| |
| return gcvSTATUS_NOT_FOUND; |
| } |
| |