| /**************************************************************************** |
| * |
| * The MIT License (MIT) |
| * |
| * Copyright (c) 2014 - 2020 Vivante Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in |
| * all copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| * DEALINGS IN THE SOFTWARE. |
| * |
| ***************************************************************************** |
| * |
| * The GPL License (GPL) |
| * |
| * Copyright (C) 2014 - 2020 Vivante Corporation |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; either version 2 |
| * of the License, or (at your option) any later version. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| * |
| ***************************************************************************** |
| * |
| * Note: This software is released under dual MIT and GPL licenses. A |
| * recipient may use this file under the terms of either the MIT license or |
| * GPL License. If you wish to use only one license not the other, you can |
| * indicate your decision by deleting one of the above license notices in your |
| * version of this file. |
| * |
| *****************************************************************************/ |
| |
| |
| #include "gc_hal_kernel_precomp.h" |
| #include "gc_hal_kernel_context.h" |
| |
| #define _GC_OBJ_ZONE gcvZONE_COMMAND |
| |
| /******************************************************************************\ |
| ********************************* Support Code ********************************* |
| \******************************************************************************/ |
| |
| /******************************************************************************* |
| ** |
| ** _NewQueue |
| ** |
| ** Allocate a new command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctBOOL Stalled |
| ** Indicate if hardware is stalled already. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckCOMMAND Command |
| ** gckCOMMAND object has been updated with a new command queue. |
| */ |
| static gceSTATUS |
| _NewQueue( |
| IN OUT gckCOMMAND Command, |
| IN gctBOOL Stalled |
| ) |
| { |
| gceSTATUS status; |
| gctINT currentIndex, newIndex; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| /* Switch to the next command buffer. */ |
| currentIndex = Command->index; |
| newIndex = (currentIndex + 1) % gcdCOMMAND_QUEUES; |
| |
| /* Wait for availability. */ |
| gcmkDUMP(Command->os, "#[kernel.waitsignal]"); |
| |
| gcmkONERROR(gckOS_WaitSignal( |
| Command->os, |
| Command->queues[newIndex].signal, |
| gcvFALSE, |
| gcvINFINITE |
| )); |
| |
| #if gcmIS_DEBUG(gcdDEBUG_TRACE) |
| if (newIndex < currentIndex) |
| { |
| Command->wrapCount += 1; |
| |
| gcmkTRACE_ZONE_N( |
| gcvLEVEL_INFO, gcvZONE_COMMAND, |
| 2 * 4, |
| "%s(%d): queue array wrapped around.\n", |
| __FUNCTION__, __LINE__ |
| ); |
| } |
| |
| gcmkTRACE_ZONE_N( |
| gcvLEVEL_INFO, gcvZONE_COMMAND, |
| 3 * 4, |
| "%s(%d): total queue wrap arounds %d.\n", |
| __FUNCTION__, __LINE__, Command->wrapCount |
| ); |
| |
| gcmkTRACE_ZONE_N( |
| gcvLEVEL_INFO, gcvZONE_COMMAND, |
| 3 * 4, |
| "%s(%d): switched to queue %d.\n", |
| __FUNCTION__, __LINE__, newIndex |
| ); |
| #endif |
| |
| /* Update gckCOMMAND object with new command queue. */ |
| Command->index = newIndex; |
| Command->newQueue = gcvTRUE; |
| Command->videoMem = Command->queues[newIndex].videoMem; |
| Command->logical = Command->queues[newIndex].logical; |
| Command->address = Command->queues[newIndex].address; |
| Command->offset = 0; |
| |
| if (currentIndex != -1) |
| { |
| if (Stalled) |
| { |
| gckOS_Signal( |
| Command->os, |
| Command->queues[currentIndex].signal, |
| gcvTRUE |
| ); |
| } |
| else |
| { |
| /* Mark the command queue as available. */ |
| gcmkONERROR(gckEVENT_Signal( |
| Command->kernel->eventObj, |
| Command->queues[currentIndex].signal, |
| gcvKERNEL_COMMAND |
| )); |
| } |
| } |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("Command->index=%d", Command->index); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _IncrementCommitAtom( |
| IN gckCOMMAND Command, |
| IN gctBOOL Increment |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctINT32 atomValue; |
| gctBOOL powerAcquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| /* Extract the gckHARDWARE and gckEVENT objects. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Grab the power mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, hardware->powerMutex, gcvINFINITE |
| )); |
| powerAcquired = gcvTRUE; |
| |
| /* Increment the commit atom. */ |
| if (Increment) |
| { |
| gcmkONERROR(gckOS_AtomIncrement( |
| Command->os, Command->atomCommit, &atomValue |
| )); |
| } |
| else |
| { |
| gcmkONERROR(gckOS_AtomDecrement( |
| Command->os, Command->atomCommit, &atomValue |
| )); |
| } |
| |
| /* Release the power mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex( |
| Command->os, hardware->powerMutex |
| )); |
| powerAcquired = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (powerAcquired) |
| { |
| /* Release the power mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex( |
| Command->os, hardware->powerMutex |
| )); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _CheckFlushMMU( |
| IN gckCOMMAND Command, |
| IN gckHARDWARE Hardware |
| ) |
| { |
| #if gcdSECURITY |
| return gcvSTATUS_OK; |
| #else |
| gceSTATUS status; |
| gctUINT32 oldValue; |
| gctBOOL pause = gcvFALSE; |
| |
| gctUINT8_PTR pointer; |
| gctUINT32 address; |
| gctUINT32 eventBytes; |
| gctUINT32 endBytes; |
| gctUINT32 bufferSize; |
| gctUINT32 executeBytes; |
| |
| gckOS_AtomicExchange(Command->os, |
| Hardware->pageTableDirty[gcvENGINE_RENDER], |
| 0, |
| &oldValue); |
| |
| if (oldValue) |
| { |
| /* Page Table is upated, flush mmu before commit. */ |
| gctUINT32 flushBytes; |
| |
| gcmkONERROR(gckHARDWARE_FlushMMU( |
| Hardware, |
| gcvNULL, |
| gcvINVALID_ADDRESS, |
| 0, |
| &flushBytes |
| )); |
| |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| flushBytes, |
| (gctPOINTER *)&pointer, |
| &bufferSize |
| )); |
| |
| /* Pointer to reserved address. */ |
| address = Command->address + Command->offset; |
| |
| /* |
| * subsequent 8 bytes are wait-link commands. |
| * Set more existed bytes for now. |
| */ |
| gcmkONERROR(gckHARDWARE_FlushMMU( |
| Hardware, |
| pointer, |
| address, |
| (bufferSize - flushBytes), |
| &flushBytes |
| )); |
| |
| gcmkONERROR(gckCOMMAND_Execute(Command, flushBytes)); |
| |
| if ((oldValue & gcvPAGE_TABLE_DIRTY_BIT_FE) |
| && (!Hardware->stallFEPrefetch) |
| ) |
| { |
| pause = gcvTRUE; |
| } |
| } |
| |
| if (pause) |
| { |
| /* Query size. */ |
| gcmkONERROR(gckWLFE_Event(Hardware, gcvNULL, 0, gcvKERNEL_PIXEL, &eventBytes)); |
| gcmkONERROR(gckWLFE_End(Hardware, gcvNULL, ~0U, &endBytes)); |
| |
| executeBytes = eventBytes + endBytes; |
| |
| /* Reserve space. */ |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| executeBytes, |
| (gctPOINTER *)&pointer, |
| &bufferSize |
| )); |
| |
| /* Pointer to reserved address. */ |
| address = Command->address + Command->offset; |
| |
| /* Append EVENT(29). */ |
| gcmkONERROR(gckWLFE_Event( |
| Hardware, |
| pointer, |
| 29, |
| gcvKERNEL_PIXEL, |
| &eventBytes |
| )); |
| |
| /* Append END. */ |
| pointer += eventBytes; |
| address += eventBytes; |
| |
| gcmkONERROR(gckWLFE_End(Hardware, pointer, address, &endBytes)); |
| |
| gcmkONERROR(gckCOMMAND_Execute(Command, executeBytes)); |
| } |
| |
| return gcvSTATUS_OK; |
| OnError: |
| return status; |
| #endif |
| } |
| |
| /* WaitLink FE only. */ |
| static gceSTATUS |
| _DummyDraw( |
| IN gckCOMMAND Command |
| ) |
| { |
| #if gcdSECURITY |
| return gcvSTATUS_OK; |
| #else |
| gceSTATUS status; |
| gckHARDWARE hardware = Command->kernel->hardware; |
| |
| gctUINT8_PTR pointer; |
| gctUINT32 bufferSize; |
| |
| gctUINT32 dummyDrawBytes; |
| gceDUMMY_DRAW_TYPE dummyDrawType = gcvDUMMY_DRAW_INVALID; |
| |
| if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_NEED_DUMMYDRAW)) |
| { |
| dummyDrawType = gcvDUMMY_DRAW_GC400; |
| } |
| |
| if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_DEFER_FILL_FIX) && |
| gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC)) |
| { |
| dummyDrawType = gcvDUMMY_DRAW_V60; |
| } |
| |
| if (dummyDrawType != gcvDUMMY_DRAW_INVALID) |
| { |
| gckHARDWARE_DummyDraw(hardware, gcvNULL, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); |
| |
| /* Reserve space. */ |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| dummyDrawBytes, |
| (gctPOINTER *)&pointer, |
| &bufferSize |
| )); |
| |
| gckHARDWARE_DummyDraw(hardware, pointer, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); |
| |
| gcmkONERROR(gckCOMMAND_Execute(Command, dummyDrawBytes)); |
| } |
| |
| return gcvSTATUS_OK; |
| OnError: |
| return status; |
| #endif |
| } |
| |
| /* |
| * Wait pending semaphores. |
| * |
| * next == free: full, no more semaphores. |
| * (free + 1) = next: empty |
| */ |
| static gcmINLINE gceSTATUS |
| _WaitPendingMcfeSema( |
| gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| const gctUINT count = gcmCOUNTOF(Command->pendingSema); |
| gctUINT32 nextFreePos; |
| gctUINT32 timeout = gcvINFINITE; |
| |
| gcmkHEADER_ARG("freePendingPos=%u nextPendingPos=%u", |
| Command->freePendingPos, Command->nextPendingPos); |
| |
| nextFreePos = (Command->freePendingPos + 1) % count; |
| |
| if (nextFreePos == Command->nextPendingPos) |
| { |
| /* No pendings. */ |
| gcmkONERROR(gcvSTATUS_NOT_FOUND); |
| } |
| |
| while (nextFreePos != Command->nextPendingPos) |
| { |
| /* Timeout is infinite in the first to at least free one slot. */ |
| status = gckOS_WaitSignal(Command->os, |
| Command->pendingSema[nextFreePos].signal, |
| gcvFALSE, |
| timeout); |
| |
| if (status == gcvSTATUS_TIMEOUT) |
| { |
| /* Timeout out is OK for later pendings. */ |
| break; |
| } |
| |
| gcmkONERROR(status); |
| |
| /* Do not wait for the later slots. */ |
| timeout = 0; |
| |
| /* More free semaphores can be used. */ |
| Command->freeSemaId = Command->pendingSema[nextFreePos].semaId; |
| |
| /* Advance free pos. */ |
| Command->freePendingPos = nextFreePos; |
| nextFreePos = (nextFreePos + 1) % count; |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gctUINT32 |
| _GetFreeMcfeSemaNum( |
| gckCOMMAND Command |
| ) |
| { |
| gctUINT32 num = 0; |
| |
| if (Command->nextSemaId <= Command->freeSemaId) |
| { |
| num = Command->freeSemaId - Command->nextSemaId; |
| } |
| else |
| { |
| num = Command->totalSemaId - Command->nextSemaId + Command->freeSemaId; |
| } |
| |
| return num; |
| } |
| |
| /* |
| * Get next semaphore id in semaphore ring. |
| * |
| * There are semaMinThreshhold semaphores are reserved for system operations, |
| * such as _SyncToSystemChannel etc. |
| * The rest semaphores are regular ones. |
| */ |
| static gcmINLINE gceSTATUS |
| _GetNextMcfeSemaId( |
| gckCOMMAND Command, |
| gctBOOL regularSema, |
| gctUINT32 * SemaId |
| ) |
| { |
| gctUINT32 freeSemaNum = 0; |
| |
| gceSTATUS status = gcvSTATUS_OK; |
| |
| /* |
| * See the comments in struct definition. |
| * wait when run out of semaphores. |
| */ |
| freeSemaNum = _GetFreeMcfeSemaNum(Command); |
| |
| if ((regularSema && (freeSemaNum <= Command->semaMinThreshhold)) || |
| (!regularSema && (freeSemaNum == 0))) |
| { |
| gcmkONERROR(_WaitPendingMcfeSema(Command)); |
| } |
| |
| gcmkASSERT(Command->nextSemaId != Command->freeSemaId); |
| |
| /* Output the semaphore ID. */ |
| *SemaId = Command->nextSemaId; |
| |
| /* Advance to next. */ |
| if (++Command->nextSemaId == Command->totalSemaId) |
| { |
| Command->nextSemaId = 0; |
| } |
| |
| OnError: |
| return status; |
| } |
| |
| /* |
| * Get next pending pos in pending semaphore tracking ring. |
| */ |
| static gcmINLINE gceSTATUS |
| _GetNextPendingPos( |
| gckCOMMAND Command, |
| gctUINT32 * Pos |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| |
| /* Wait when out of pending ring. */ |
| if (Command->nextPendingPos == Command->freePendingPos) |
| { |
| /* Run out of pending semaphore tracking ring. */ |
| gcmkONERROR(_WaitPendingMcfeSema(Command)); |
| } |
| |
| gcmkASSERT(Command->nextPendingPos != Command->freePendingPos); |
| |
| *Pos = Command->nextPendingPos; |
| |
| /* Advance to next. */ |
| if (++Command->nextPendingPos == gcmCOUNTOF(Command->pendingSema)) |
| { |
| Command->nextPendingPos = 0; |
| } |
| |
| OnError: |
| return status; |
| } |
| |
| /* |
| * Sync specific channels to system channel. |
| * Record semaphores to pendingSema structure. |
| * 'SyncChannel' is cleared upon function return. |
| */ |
| static gceSTATUS |
| _SyncToSystemChannel( |
| gckCOMMAND Command, |
| gctUINT64 SyncChannel[2] |
| ) |
| { |
| gceSTATUS status; |
| gckKERNEL kernel = Command->kernel; |
| gckHARDWARE hardware = kernel->hardware; |
| gctUINT8 semaId[128]; |
| gctUINT32 semaCount = 0; |
| gctUINT32 reqBytes = 0; |
| gctUINT32 bytes = 0; |
| gctUINT8_PTR buffer; |
| gctUINT32 i; |
| gctUINT32 pri; |
| |
| /* Ignore system channel. */ |
| SyncChannel[0] &= ~((gctUINT64)1ull); |
| SyncChannel[1] &= ~((gctUINT64)1ull); |
| |
| if (!SyncChannel[0] && !SyncChannel[1]) |
| { |
| return gcvSTATUS_OK; |
| } |
| |
| /* Query SendSemaphore command size. */ |
| gckMCFE_SendSemaphore(hardware, gcvNULL, 0, &reqBytes); |
| |
| for (pri = 0; pri < 2; pri++) |
| { |
| for (i = 1; i < 64 && SyncChannel[pri]; i++) |
| { |
| gctUINT32 id; |
| |
| if (!(SyncChannel[pri] & (1ull << i))) |
| { |
| continue; |
| } |
| |
| /* Get a free semaphore id. */ |
| gcmkONERROR(_GetNextMcfeSemaId(Command, gcvFALSE, &id)); |
| semaId[semaCount++] = (gctUINT8)id; |
| |
| gcmkONERROR(gckCOMMAND_Reserve(Command, reqBytes, (gctPOINTER *)&buffer, &bytes)); |
| |
| /* Send semaphore executed in specified channel. */ |
| gckMCFE_SendSemaphore(hardware, buffer, id, &bytes); |
| |
| gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, pri, i, reqBytes)); |
| |
| /* Remove the sync'ed channel. */ |
| SyncChannel[pri] &= ~(1ull << i); |
| } |
| } |
| |
| if (semaCount > 0) |
| { |
| gctUINT32 pos = 0; |
| gckEVENT eventObj = kernel->eventObj; |
| gctUINT32 bufferLen = 0; |
| |
| /* Query WaitSemaphore command size. */ |
| gckMCFE_WaitSemaphore(hardware, gcvNULL, 0, &reqBytes); |
| reqBytes *= semaCount; |
| |
| gcmkONERROR(gckCOMMAND_Reserve(Command, reqBytes, (gctPOINTER *)&buffer, &bufferLen)); |
| |
| for (i = 0; i < semaCount; i++) |
| { |
| bytes = bufferLen; |
| |
| /* Wait semaphores executed in fixed system channel. */ |
| gckMCFE_WaitSemaphore(hardware, buffer, semaId[i], &bytes); |
| |
| buffer += bytes; |
| bufferLen -= bytes; |
| } |
| |
| gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, 0, 0, reqBytes)); |
| |
| /* Now upload the pending semaphore tracking ring. */ |
| gcmkONERROR(_GetNextPendingPos(Command, &pos)); |
| |
| /* Update latest pending semaphore id. */ |
| Command->pendingSema[pos].semaId = (gctUINT32)semaId[semaCount - 1]; |
| |
| /* Send the signal by event. */ |
| gcmkONERROR(gckEVENT_Signal( |
| eventObj, |
| Command->pendingSema[pos].signal, |
| gcvKERNEL_PIXEL |
| )); |
| |
| gcmkONERROR(gckEVENT_Submit( |
| eventObj, |
| gcvTRUE, |
| gcvFALSE |
| )); |
| } |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| return status; |
| } |
| |
| static gcmINLINE gceSTATUS |
| _SyncFromSystemChannel( |
| gckCOMMAND Command, |
| gctBOOL Priority, |
| gctUINT32 ChannelId |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware = Command->kernel->hardware; |
| gctUINT32 reqBytes = 0; |
| gctUINT32 bytes = 0; |
| gctUINT8_PTR buffer; |
| gctUINT32 id; |
| |
| if (!(Command->syncChannel[Priority ? 1 : 0] & (1ull << ChannelId))) |
| { |
| /* No need to sync. */ |
| return gcvSTATUS_OK; |
| } |
| |
| /* Get a semaphore. */ |
| gcmkONERROR(_GetNextMcfeSemaId(Command, gcvFALSE, &id)); |
| |
| /* Send the semaphore in system channel. */ |
| gckMCFE_SendSemaphore(hardware, gcvNULL, 0, &reqBytes); |
| |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| reqBytes, |
| (gctPOINTER *)&buffer, |
| &bytes |
| )); |
| |
| gckMCFE_SendSemaphore(hardware, buffer, id, &bytes); |
| |
| gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, gcvFALSE, 0, reqBytes)); |
| |
| /* Wait the semaphore in specific channel. */ |
| gckMCFE_WaitSemaphore(hardware, gcvNULL, 0, &reqBytes); |
| |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| reqBytes, |
| (gctPOINTER *)&buffer, |
| &bytes |
| )); |
| |
| gckMCFE_WaitSemaphore(hardware, buffer, id, &bytes); |
| |
| gcmkONERROR(gckCOMMAND_ExecuteMultiChannel( |
| Command, |
| Priority, |
| ChannelId, |
| reqBytes |
| )); |
| |
| /* Clear the sync flag. */ |
| Command->syncChannel[Priority ? 1 : 0] &= ~(1ull << ChannelId); |
| |
| /* Can not track the semaphore here. */ |
| return gcvSTATUS_OK; |
| |
| OnError: |
| return status; |
| } |
| |
| static gceSTATUS |
| _CheckFlushMcfeMMU( |
| IN gckCOMMAND Command, |
| IN gckHARDWARE Hardware |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gctUINT32 oldValue; |
| gctUINT32 reqBytes; |
| gctUINT32 bytes; |
| gctUINT8_PTR buffer; |
| gctUINT32 id = 0; |
| |
| gckOS_AtomicExchange(Command->os, |
| Hardware->pageTableDirty[gcvENGINE_RENDER], |
| 0, |
| &oldValue); |
| |
| if (!oldValue) |
| { |
| return gcvSTATUS_OK; |
| } |
| |
| /* |
| * We had sync'ed to system channel in every commit, see comments in Commit. |
| * It should not run into sync again here, unless there's some other place |
| * causes channels dirty. Let's check it here. |
| */ |
| gcmkONERROR(_SyncToSystemChannel(Command, Command->dirtyChannel)); |
| |
| /* Query flush Mcfe MMU cache command bytes. */ |
| gcmkONERROR(gckHARDWARE_FlushMcfeMMU(Hardware, gcvNULL, &reqBytes)); |
| |
| /* Query semaphore command bytes. */ |
| gcmkONERROR( |
| gckMCFE_SendSemaphore(Hardware, gcvNULL, 0, &bytes)); |
| reqBytes += bytes; |
| |
| gcmkONERROR( |
| gckMCFE_WaitSemaphore(Hardware, gcvNULL, 0, &bytes)); |
| reqBytes += bytes; |
| |
| /* Get a semaphore. */ |
| gcmkONERROR(_GetNextMcfeSemaId(Command, gcvFALSE, &id)); |
| |
| /* Request command buffer for system channel. */ |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| reqBytes, |
| (gctPOINTER *)&buffer, |
| &bytes |
| )); |
| |
| /* Do flush mmu. */ |
| gckHARDWARE_FlushMcfeMMU(Hardware, buffer, &bytes); |
| buffer += bytes; |
| |
| /* Send and wait semaphore in the system channel itself. */ |
| gcmkONERROR(gckMCFE_SendSemaphore(Hardware, buffer, id, &bytes)); |
| buffer += bytes; |
| |
| gcmkONERROR(gckMCFE_WaitSemaphore(Hardware, buffer, id, &bytes)); |
| |
| /* Execute flush mmu and send semaphores. */ |
| gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, 0, 0, reqBytes)); |
| |
| /* Need sync from system channel. */ |
| Command->syncChannel[0] = ~1ull; |
| Command->syncChannel[1] = ~1ull; |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| return status; |
| } |
| |
| /* |
| * Find sema id from the map. |
| * Returns semaphore Id, ie the array index of semaHandleMap. |
| * -1 if not found. |
| */ |
| static gcmINLINE gctINT32 |
| _FindSemaIdFromMap( |
| IN gckCOMMAND Command, |
| IN gctUINT32 SemaHandle |
| ) |
| { |
| gctUINT32 semaId = Command->nextSemaId; |
| |
| do |
| { |
| /* |
| * Only need to check semaId between signaledId (inclusive) and |
| * nextSemaId (exclusive). |
| */ |
| semaId = (semaId == 0) ? (Command->totalSemaId - 1) : (semaId - 1); |
| |
| if (Command->semaHandleMap[semaId] == SemaHandle) |
| { |
| return (gctINT32)semaId; |
| } |
| } |
| while (semaId != Command->freeSemaId); |
| |
| return -1; |
| } |
| |
| /* Put together patch list handling variables. */ |
| typedef struct _gcsPATCH_LIST_VARIABLE |
| { |
| /* gcvHAL_PATCH_VIDMEM_TIMESTAMP. */ |
| gctUINT64 maxAsyncTimestamp; |
| |
| /* gcvHAL_PATCH_MCFE_SEMAPHORE. */ |
| gctBOOL semaUsed; |
| } |
| gcsPATCH_LIST_VARIABLE; |
| |
| /* Patch item handler typedef. */ |
| typedef gceSTATUS |
| (* PATCH_ITEM_HANDLER)( |
| IN gckCOMMAND Command, |
| IN gcsHAL_COMMAND_LOCATION * CommandBuffer, |
| IN gctPOINTER Patch, |
| IN gcsPATCH_LIST_VARIABLE * PatchListVar |
| ); |
| |
| static const gctUINT32 _PatchItemSize[] = |
| { |
| 0, |
| (gctUINT32)sizeof(gcsHAL_PATCH_VIDMEM_ADDRESS), |
| (gctUINT32)sizeof(gcsHAL_PATCH_MCFE_SEMAPHORE), |
| (gctUINT32)sizeof(gcsHAL_PATCH_VIDMEM_TIMESTAMP), |
| }; |
| |
| static gceSTATUS |
| _HandleVidmemAddressPatch( |
| IN gckCOMMAND Command, |
| IN gcsHAL_COMMAND_LOCATION * CommandBuffer, |
| IN gctPOINTER Patch, |
| IN gcsPATCH_LIST_VARIABLE * PatchListVar |
| ) |
| { |
| gceSTATUS status = gcvSTATUS_OK; |
| gcsHAL_PATCH_VIDMEM_ADDRESS * patch = Patch; |
| |
| gcmkHEADER_ARG("Command=%p location=0x%x node=0x%x offset=%x", |
| Command, patch->location, patch->node, patch->offset); |
| |
| (void)status; |
| (void)patch; |
| |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| } |
| |
| static gceSTATUS |
| _HandleMCFESemaphorePatch( |
| IN gckCOMMAND Command, |
| IN gcsHAL_COMMAND_LOCATION * CommandBuffer, |
| IN gctPOINTER Patch, |
| IN gcsPATCH_LIST_VARIABLE * PatchListVar |
| ) |
| { |
| gckHARDWARE hardware = Command->kernel->hardware; |
| gctINT32 index; |
| gctUINT32 semaId; |
| gceSTATUS status; |
| gctUINT32 bytes = 8; |
| gctUINT32 buffer[2]; |
| gctUINT8_PTR location; |
| gcsHAL_PATCH_MCFE_SEMAPHORE * patch = (gcsHAL_PATCH_MCFE_SEMAPHORE *)Patch; |
| |
| gcmkHEADER_ARG("Command=%p location=0x%x semaHandle=%d", |
| Command, patch->location, patch->semaHandle); |
| |
| index = _FindSemaIdFromMap(Command, patch->semaHandle); |
| |
| if (index < 0) |
| { |
| status = _GetNextMcfeSemaId(Command, gcvTRUE, &semaId); |
| |
| if (gcmIS_ERROR(status)) |
| { |
| gcmkONERROR(_SyncToSystemChannel(Command, Command->dirtyChannel)); |
| gcmkONERROR(_GetNextMcfeSemaId(Command, gcvTRUE, &semaId)); |
| } |
| |
| Command->semaHandleMap[semaId] = patch->semaHandle; |
| } |
| else |
| { |
| semaId = (gctUINT32)index; |
| |
| /* One send must match one wait, will assign new id next time. */ |
| Command->semaHandleMap[semaId] = 0; |
| } |
| |
| if (patch->sendSema) |
| { |
| gcmkONERROR(gckMCFE_SendSemaphore(hardware, buffer, semaId, &bytes)); |
| } |
| else |
| { |
| gcmkONERROR(gckMCFE_WaitSemaphore(hardware, buffer, semaId, &bytes)); |
| } |
| |
| gcmkASSERT(bytes == 8); |
| |
| location = gcmUINT64_TO_PTR(CommandBuffer->logical + patch->location); |
| |
| /* Patch the command buffer. */ |
| gckOS_WriteMemory(Command->os, location, buffer[0]); |
| gckOS_WriteMemory(Command->os, location + 4, buffer[1]); |
| |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _HandleTimestampPatch( |
| IN gckCOMMAND Command, |
| IN gcsHAL_COMMAND_LOCATION * CommandBuffer, |
| IN gctPOINTER Patch, |
| IN gcsPATCH_LIST_VARIABLE * PatchListVar |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 processID; |
| gckVIDMEM_NODE videoMem = gcvNULL; |
| gcsHAL_PATCH_VIDMEM_TIMESTAMP * patch = Patch; |
| gceENGINE engine = Command->feType == gcvHW_FE_ASYNC ? gcvENGINE_BLT |
| : gcvENGINE_RENDER; |
| |
| gcmkHEADER_ARG("Command=%p node=0x%x", Command, patch->handle); |
| |
| /* Get the current process ID. */ |
| gcmkONERROR(gckOS_GetProcessID(&processID)); |
| |
| gcmkONERROR( |
| gckVIDMEM_HANDLE_Lookup(Command->kernel, |
| processID, |
| patch->handle, |
| &videoMem)); |
| |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Reference(Command->kernel, videoMem)); |
| |
| /* Touch video memory node. */ |
| gcmkVERIFY_OK( |
| gckVIDMEM_NODE_SetCommitStamp(Command->kernel, |
| engine, |
| videoMem, |
| Command->commitStamp)); |
| |
| if ((engine == gcvENGINE_RENDER) && Command->kernel->asyncCommand) |
| { |
| /* Find the latest timestamp of the nodes used in async FE. */ |
| gctUINT64 stamp = 0; |
| |
| /* Get stamp touched async command buffer. */ |
| gcmkVERIFY_OK( |
| gckVIDMEM_NODE_GetCommitStamp(Command->kernel, |
| gcvENGINE_BLT, |
| videoMem, |
| &stamp)); |
| |
| /* Find latest one. */ |
| if (PatchListVar->maxAsyncTimestamp < stamp) |
| { |
| PatchListVar->maxAsyncTimestamp = stamp; |
| } |
| } |
| |
| OnError: |
| if (videoMem) |
| { |
| gckVIDMEM_NODE_Dereference(Command->kernel, videoMem); |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| static gceSTATUS |
| _HandlePatchListSingle( |
| IN gckCOMMAND Command, |
| IN gcsHAL_COMMAND_LOCATION * CommandBuffer, |
| IN gcsHAL_PATCH_LIST * PatchList, |
| IN gctBOOL NeedCopy, |
| IN gcsPATCH_LIST_VARIABLE * PatchListVar |
| ) |
| { |
| gceSTATUS status; |
| /* 256 bytes for storage. */ |
| gctUINT64 storage[32]; |
| gctPOINTER kArray = gcvNULL; |
| gctPOINTER userPtr = gcvNULL; |
| gctUINT32 index = 0; |
| gctUINT32 count = 0; |
| gctUINT32 itemSize = 0; |
| gctUINT32 batchCount = 0; |
| |
| static const PATCH_ITEM_HANDLER patchHandler[] = |
| { |
| gcvNULL, |
| _HandleVidmemAddressPatch, |
| _HandleMCFESemaphorePatch, |
| _HandleTimestampPatch, |
| }; |
| PATCH_ITEM_HANDLER handler; |
| |
| gcmkHEADER_ARG("Command=%p CommandBuffer=%p PatchList=%p type=%d", |
| Command, CommandBuffer, PatchList, PatchList->type); |
| |
| if (PatchList->type >= gcmCOUNTOF(_PatchItemSize) || PatchList->type >= gcmCOUNTOF(patchHandler)) |
| { |
| /* Exceeds buffer max size. */ |
| gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); |
| } |
| |
| itemSize = _PatchItemSize[PatchList->type]; |
| |
| batchCount = (gctUINT32)(sizeof(storage) / itemSize); |
| |
| handler = patchHandler[PatchList->type]; |
| |
| while (index < PatchList->count) |
| { |
| gctUINT i; |
| gctUINT8_PTR ptr; |
| |
| /* Determine batch count, don't handle too many in one batch. */ |
| count = PatchList->count - index; |
| |
| if (count > batchCount) |
| { |
| count = batchCount; |
| } |
| |
| userPtr = gcmUINT64_TO_PTR(PatchList->patchArray + itemSize * index); |
| |
| /* Copy/map a patch array batch from user. */ |
| if (NeedCopy) |
| { |
| kArray = storage; |
| |
| status = gckOS_CopyFromUserData( |
| Command->os, |
| kArray, |
| userPtr, |
| itemSize * count |
| ); |
| } |
| else |
| { |
| status = gckOS_MapUserPointer( |
| Command->os, |
| userPtr, |
| itemSize * count, |
| (gctPOINTER *)&kArray |
| ); |
| } |
| |
| if (gcmIS_ERROR(status)) |
| { |
| userPtr = gcvNULL; |
| gcmkONERROR(status); |
| } |
| |
| /* Advance to next batch. */ |
| index += count; |
| |
| ptr = (gctUINT8_PTR)kArray; |
| |
| for (i = 0; i < count; i++) |
| { |
| /* Call handler. */ |
| gcmkONERROR( |
| handler(Command, CommandBuffer, ptr, PatchListVar)); |
| |
| /* Advance to next patch. */ |
| ptr += itemSize; |
| } |
| |
| /* Unmap user pointer if mapped. */ |
| if (!NeedCopy) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| userPtr, |
| itemSize * count, |
| kArray |
| )); |
| } |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (!NeedCopy && userPtr) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| userPtr, |
| itemSize * count, |
| kArray |
| )); |
| |
| userPtr = gcvNULL; |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _HandlePatchList( |
| IN gckCOMMAND Command, |
| IN gcsHAL_COMMAND_LOCATION * CommandBuffer, |
| OUT gcsPATCH_LIST_VARIABLE * PatchListVar |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL needCopy = gcvFALSE; |
| gcsHAL_PATCH_LIST storage; |
| gcsHAL_PATCH_LIST * kPatchList = gcvNULL; |
| gctPOINTER userPtr = gcmUINT64_TO_PTR(CommandBuffer->patchHead); |
| |
| gcmkHEADER_ARG("Command=%p CommandBuffer=%p", Command, CommandBuffer); |
| |
| /* Check wehther we need to copy the structures or not. */ |
| gcmkONERROR(gckOS_QueryNeedCopy(Command->os, 0, &needCopy)); |
| |
| while (userPtr) |
| { |
| gctUINT64 next; |
| |
| /* Copy/map a patch from user. */ |
| if (needCopy) |
| { |
| kPatchList = &storage; |
| |
| status = gckOS_CopyFromUserData( |
| Command->os, |
| kPatchList, |
| userPtr, |
| sizeof(gcsHAL_PATCH_LIST) |
| ); |
| } |
| else |
| { |
| status = gckOS_MapUserPointer( |
| Command->os, |
| userPtr, |
| sizeof(gcsHAL_PATCH_LIST), |
| (gctPOINTER *)&kPatchList |
| ); |
| } |
| |
| if (gcmIS_ERROR(status)) |
| { |
| userPtr = gcvNULL; |
| gcmkONERROR(status); |
| } |
| |
| /* Do handle patch. */ |
| gcmkASSERT(kPatchList->type < gcvHAL_PATCH_TYPE_COUNT); |
| |
| gcmkONERROR( |
| _HandlePatchListSingle(Command, |
| CommandBuffer, |
| kPatchList, |
| needCopy, |
| PatchListVar)); |
| |
| next = kPatchList->next; |
| |
| /* Unmap user pointer if mapped. */ |
| if (!needCopy) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| userPtr, |
| sizeof(gcsHAL_PATCH_LIST), |
| kPatchList |
| )); |
| } |
| |
| /* Advance to next patch from user. */ |
| userPtr = gcmUINT64_TO_PTR(next); |
| } |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (!needCopy && userPtr) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| userPtr, |
| sizeof(gcsHAL_PATCH_LIST), |
| kPatchList |
| )); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _WaitForAsyncCommandStamp( |
| IN gckCOMMAND Command, |
| IN gctUINT64 Stamp |
| ) |
| { |
| gctUINT32 bytes; |
| gceSTATUS status; |
| gctUINT32 fenceAddress; |
| gctUINT32 bufferSize; |
| gctPOINTER pointer; |
| gckCOMMAND asyncCommand = Command->kernel->asyncCommand; |
| |
| gcmkHEADER_ARG("Stamp = 0x%llx", Stamp); |
| |
| if (*(gctUINT64 *)asyncCommand->fence->logical >= Stamp) |
| { |
| /* Already satisfied, skip. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| fenceAddress = asyncCommand->fence->address; |
| |
| gcmkONERROR(gckHARDWARE_WaitFence( |
| Command->kernel->hardware, |
| gcvNULL, |
| Stamp, |
| fenceAddress, |
| &bytes |
| )); |
| |
| gcmkONERROR(gckCOMMAND_Reserve( |
| Command, |
| bytes, |
| &pointer, |
| &bufferSize |
| )); |
| |
| gcmkONERROR(gckHARDWARE_WaitFence( |
| Command->kernel->hardware, |
| pointer, |
| Stamp, |
| fenceAddress, |
| &bytes |
| )); |
| |
| gcmkONERROR(gckCOMMAND_Execute(Command, bytes)); |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************\ |
| ****************************** gckCOMMAND API Code ****************************** |
| \******************************************************************************/ |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Construct |
| ** |
| ** Construct a new gckCOMMAND object. |
| ** |
| ** INPUT: |
| ** |
| ** gckKERNEL Kernel |
| ** Pointer to an gckKERNEL object. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckCOMMAND * Command |
| ** Pointer to a variable that will hold the pointer to the gckCOMMAND |
| ** object. |
| */ |
| gceSTATUS |
| gckCOMMAND_Construct( |
| IN gckKERNEL Kernel, |
| IN gceHW_FE_TYPE FeType, |
| OUT gckCOMMAND * Command |
| ) |
| { |
| gckOS os; |
| gckCOMMAND command = gcvNULL; |
| gceSTATUS status; |
| gctINT i; |
| gctPOINTER pointer = gcvNULL; |
| gctSIZE_T pageSize; |
| |
| gcmkHEADER_ARG("Kernel=%p", Kernel); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); |
| gcmkVERIFY_ARGUMENT(Command != gcvNULL); |
| |
| /* Extract the gckOS object. */ |
| os = Kernel->os; |
| |
| /* Allocate the gckCOMMAND structure. */ |
| gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckCOMMAND), &pointer)); |
| command = pointer; |
| |
| /* Reset the entire object. */ |
| gcmkONERROR(gckOS_ZeroMemory(command, gcmSIZEOF(struct _gckCOMMAND))); |
| |
| /* Initialize the gckCOMMAND object.*/ |
| command->object.type = gcvOBJ_COMMAND; |
| command->kernel = Kernel; |
| command->os = os; |
| |
| command->feType = FeType; |
| |
| /* Get the command buffer requirements. */ |
| gcmkONERROR(gckHARDWARE_QueryCommandBuffer( |
| Kernel->hardware, |
| gcvENGINE_RENDER, |
| &command->alignment, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| /* Create the command queue mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &command->mutexQueue)); |
| |
| /* Create the context switching mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContext)); |
| |
| /* Create the context switching mutex. */ |
| gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContextSeq)); |
| |
| /* Create the power management semaphore. */ |
| gcmkONERROR(gckOS_CreateSemaphore(os, &command->powerSemaphore)); |
| |
| /* Create the commit atom. */ |
| gcmkONERROR(gckOS_AtomConstruct(os, &command->atomCommit)); |
| |
| /* Get the page size from teh OS. */ |
| gcmkONERROR(gckOS_GetPageSize(os, &pageSize)); |
| |
| gcmkSAFECASTSIZET(command->pageSize, pageSize); |
| |
| /* Get process ID. */ |
| gcmkONERROR(gckOS_GetProcessID(&command->kernelProcessID)); |
| |
| /* Set hardware to pipe 0. */ |
| command->pipeSelect = gcvPIPE_INVALID; |
| |
| /* Pre-allocate the command queues. */ |
| for (i = 0; i < gcdCOMMAND_QUEUES; ++i) |
| { |
| #if !gcdCAPTURE_ONLY_MODE |
| gcePOOL pool = gcvPOOL_DEFAULT; |
| #else |
| gcePOOL pool = gcvPOOL_VIRTUAL; |
| #endif |
| |
| gctSIZE_T size = pageSize; |
| gckVIDMEM_NODE videoMem = gcvNULL; |
| gctUINT32 allocFlag = 0; |
| |
| #if gcdENABLE_CACHEABLE_COMMAND_BUFFER |
| allocFlag = gcvALLOC_FLAG_CACHEABLE; |
| #endif |
| |
| /* Allocate video memory node for command buffers. */ |
| gcmkONERROR(gckKERNEL_AllocateVideoMemory( |
| Kernel, |
| 64, |
| gcvVIDMEM_TYPE_COMMAND, |
| allocFlag, |
| &size, |
| &pool, |
| &videoMem |
| )); |
| |
| command->queues[i].videoMem = videoMem; |
| |
| /* Lock for GPU access. */ |
| gcmkONERROR(gckVIDMEM_NODE_Lock( |
| Kernel, |
| videoMem, |
| &command->queues[i].address |
| )); |
| |
| /* Lock for kernel side CPU access. */ |
| gcmkONERROR(gckVIDMEM_NODE_LockCPU( |
| Kernel, |
| videoMem, |
| gcvFALSE, |
| gcvFALSE, |
| &command->queues[i].logical |
| )); |
| |
| gcmkONERROR(gckOS_CreateSignal( |
| os, gcvFALSE, &command->queues[i].signal |
| )); |
| |
| gcmkONERROR(gckOS_Signal( |
| os, command->queues[i].signal, gcvTRUE |
| )); |
| } |
| |
| #if gcdRECORD_COMMAND |
| gcmkONERROR(gckRECORDER_Construct(os, Kernel->hardware, &command->recorder)); |
| #endif |
| |
| gcmkONERROR(gckFENCE_Create( |
| os, Kernel, &command->fence |
| )); |
| |
| /* No command queue in use yet. */ |
| command->index = -1; |
| command->logical = gcvNULL; |
| command->newQueue = gcvFALSE; |
| |
| /* Query mcfe semaphore count. */ |
| if (FeType == gcvHW_FE_MULTI_CHANNEL) |
| { |
| command->totalSemaId = 128; |
| |
| /* Empty sema id ring. */ |
| command->nextSemaId = 0; |
| command->freeSemaId = command->totalSemaId - 1; |
| |
| command->semaMinThreshhold = 16; |
| |
| /* Create signals. */ |
| for (i = 0; i < (gctINT)gcmCOUNTOF(command->pendingSema); i++) |
| { |
| gcmkONERROR(gckOS_CreateSignal( |
| os, |
| gcvFALSE, |
| &command->pendingSema[i].signal |
| )); |
| } |
| |
| /* Empty pending sema tracking ring. */ |
| command->nextPendingPos = 0; |
| command->freePendingPos = gcmCOUNTOF(command->pendingSema) - 1; |
| |
| /* Allocate sema handle mapping. */ |
| gcmkONERROR(gckOS_Allocate( |
| os, |
| command->totalSemaId * sizeof(gctUINT32), |
| &pointer |
| )); |
| |
| command->semaHandleMap = (gctUINT32 *)pointer; |
| |
| gcmkVERIFY_OK(gckOS_ZeroMemory( |
| command->semaHandleMap, |
| command->totalSemaId * sizeof(gctUINT32) |
| )); |
| } |
| |
| /* Command is not yet running. */ |
| command->running = gcvFALSE; |
| |
| /* Command queue is idle. */ |
| command->idle = gcvTRUE; |
| |
| /* Commit stamp start from 1. */ |
| command->commitStamp = 1; |
| |
| command->dummyDraw = gcvTRUE; |
| |
| /* Return pointer to the gckCOMMAND object. */ |
| *Command = command; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Command=0x%x", *Command); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Roll back. */ |
| if (command != gcvNULL) |
| { |
| gcmkVERIFY_OK(gckCOMMAND_Destroy(command)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Destroy |
| ** |
| ** Destroy an gckCOMMAND object. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to destroy. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Destroy( |
| IN gckCOMMAND Command |
| ) |
| { |
| gctINT i; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Stop the command queue. */ |
| gcmkVERIFY_OK(gckCOMMAND_Stop(Command)); |
| |
| for (i = 0; i < gcdCOMMAND_QUEUES; ++i) |
| { |
| if (Command->queues[i].signal) |
| { |
| gcmkVERIFY_OK(gckOS_DestroySignal( |
| Command->os, Command->queues[i].signal |
| )); |
| } |
| |
| if (Command->queues[i].logical) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( |
| Command->kernel, |
| Command->queues[i].videoMem, |
| 0, |
| gcvFALSE, |
| gcvFALSE |
| )); |
| |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( |
| Command->kernel, |
| Command->queues[i].videoMem, |
| 0, |
| gcvNULL |
| )); |
| |
| gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( |
| Command->kernel, |
| Command->queues[i].videoMem |
| )); |
| |
| Command->queues[i].videoMem = gcvNULL; |
| Command->queues[i].logical = gcvNULL; |
| } |
| } |
| |
| if (Command->mutexContext) |
| { |
| /* Delete the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContext)); |
| } |
| |
| if (Command->mutexContextSeq != gcvNULL) |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContextSeq)); |
| |
| if (Command->mutexQueue) |
| { |
| /* Delete the command queue mutex. */ |
| gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexQueue)); |
| } |
| |
| if (Command->powerSemaphore) |
| { |
| /* Destroy the power management semaphore. */ |
| gcmkVERIFY_OK(gckOS_DestroySemaphore(Command->os, Command->powerSemaphore)); |
| } |
| |
| if (Command->atomCommit) |
| { |
| /* Destroy the commit atom. */ |
| gcmkVERIFY_OK(gckOS_AtomDestroy(Command->os, Command->atomCommit)); |
| } |
| |
| #if gcdRECORD_COMMAND |
| gckRECORDER_Destory(Command->os, Command->recorder); |
| #endif |
| |
| if (Command->stateMap) |
| { |
| gcmkOS_SAFE_FREE(Command->os, Command->stateMap); |
| } |
| |
| if (Command->semaHandleMap) |
| { |
| gcmkOS_SAFE_FREE(Command->os, Command->semaHandleMap); |
| } |
| |
| if (Command->fence) |
| { |
| gcmkVERIFY_OK(gckFENCE_Destory(Command->os, Command->fence)); |
| } |
| |
| /* Mark object as unknown. */ |
| Command->object.type = gcvOBJ_UNKNOWN; |
| |
| /* Free the gckCOMMAND object. */ |
| gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, Command)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_EnterCommit |
| ** |
| ** Acquire command queue synchronization objects. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to destroy. |
| ** |
| ** gctBOOL FromPower |
| ** Determines whether the call originates from inside the power |
| ** management or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_EnterCommit( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctBOOL atomIncremented = gcvFALSE; |
| gctBOOL semaAcquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| /* Extract the gckHARDWARE and gckEVENT objects. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| if (!FromPower) |
| { |
| /* Increment COMMIT atom to let power management know that a commit is |
| ** in progress. */ |
| gcmkONERROR(_IncrementCommitAtom(Command, gcvTRUE)); |
| atomIncremented = gcvTRUE; |
| |
| /* Notify the system the GPU has a commit. */ |
| gcmkONERROR(gckOS_Broadcast(Command->os, |
| hardware, |
| gcvBROADCAST_GPU_COMMIT)); |
| |
| /* Acquire the power management semaphore. */ |
| gcmkONERROR(gckOS_AcquireSemaphore(Command->os, |
| Command->powerSemaphore)); |
| semaAcquired = gcvTRUE; |
| } |
| |
| /* Grab the conmmand queue mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex(Command->os, |
| Command->mutexQueue, |
| gcvINFINITE)); |
| |
| /* Success. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (semaAcquired) |
| { |
| /* Release the power management semaphore. */ |
| gcmkVERIFY_OK(gckOS_ReleaseSemaphore( |
| Command->os, Command->powerSemaphore |
| )); |
| } |
| |
| if (atomIncremented) |
| { |
| /* Decrement the commit atom. */ |
| gcmkVERIFY_OK(_IncrementCommitAtom( |
| Command, gcvFALSE |
| )); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_ExitCommit |
| ** |
| ** Release command queue synchronization objects. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to destroy. |
| ** |
| ** gctBOOL FromPower |
| ** Determines whether the call originates from inside the power |
| ** management or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_ExitCommit( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower |
| ) |
| { |
| gceSTATUS status; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| /* Release the power mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); |
| |
| if (!FromPower) |
| { |
| /* Release the power management semaphore. */ |
| gcmkONERROR(gckOS_ReleaseSemaphore(Command->os, |
| Command->powerSemaphore)); |
| |
| /* Decrement the commit atom. */ |
| gcmkONERROR(_IncrementCommitAtom(Command, gcvFALSE)); |
| } |
| |
| /* Success. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _StartWaitLinkFE( |
| IN gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctUINT32 waitOffset = 0; |
| gctUINT32 waitLinkBytes; |
| gctPOINTER logical; |
| gctUINT32 address; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| if (Command->running) |
| { |
| /* Command queue already running. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| } |
| |
| /* Extract the gckHARDWARE object. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Query the size of WAIT/LINK command sequence. */ |
| gcmkONERROR(gckWLFE_WaitLink( |
| hardware, |
| gcvNULL, |
| ~0U, |
| Command->offset, |
| &waitLinkBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| if ((Command->pageSize - Command->offset < waitLinkBytes) |
| || (Command->logical == gcvNULL) |
| ) |
| { |
| /* Start at beginning of a new queue. */ |
| gcmkONERROR(_NewQueue(Command, gcvTRUE)); |
| } |
| |
| logical = (gctUINT8_PTR) Command->logical + Command->offset; |
| address = Command->address + Command->offset; |
| |
| /* Append WAIT/LINK. */ |
| gcmkONERROR(gckWLFE_WaitLink( |
| hardware, |
| logical, |
| address, |
| 0, |
| &waitLinkBytes, |
| &waitOffset, |
| &Command->waitPos.size |
| )); |
| |
| /* Update wait command position. */ |
| Command->waitPos.videoMem = Command->videoMem; |
| Command->waitPos.offset = Command->offset + waitOffset; |
| Command->waitPos.logical = (gctUINT8_PTR) logical + waitOffset; |
| Command->waitPos.address = address + waitOffset; |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->videoMem, |
| Command->offset, |
| logical, |
| waitLinkBytes |
| )); |
| |
| /* Adjust offset. */ |
| Command->offset += waitLinkBytes; |
| Command->newQueue = gcvFALSE; |
| |
| gcmkDUMP(Command->os, "#[wait-link: fe start]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_KERNEL_COMMAND, |
| logical, |
| address, |
| waitLinkBytes |
| ); |
| |
| #if gcdSECURITY |
| /* Start FE by calling security service. */ |
| gckKERNEL_SecurityStartCommand( |
| Command->kernel |
| ); |
| #else |
| |
| #if !gcdCAPTURE_ONLY_MODE |
| /* Enable command processor. */ |
| gcmkONERROR(gckWLFE_Execute( |
| hardware, |
| address, |
| waitLinkBytes |
| )); |
| #endif |
| |
| #endif |
| |
| /* Command queue is running. */ |
| Command->running = gcvTRUE; |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _StartAsyncFE( |
| IN gckCOMMAND Command |
| ) |
| { |
| if ((Command->pageSize <= Command->offset) || |
| (Command->logical == gcvNULL)) |
| { |
| /* Start at beginning of a new queue. */ |
| gcmkVERIFY_OK(_NewQueue(Command, gcvTRUE)); |
| } |
| |
| /* Command queue is running. */ |
| Command->running = gcvTRUE; |
| |
| /* Nothing to do. */ |
| return gcvSTATUS_OK; |
| } |
| |
| static gceSTATUS |
| _StartMCFE( |
| IN gckCOMMAND Command |
| ) |
| { |
| if ((Command->pageSize <= Command->offset) || |
| (Command->logical == gcvNULL)) |
| { |
| /* Start at beginning of a new queue. */ |
| gcmkVERIFY_OK(_NewQueue(Command, gcvTRUE)); |
| } |
| |
| /* Command queue is running. */ |
| Command->running = gcvTRUE; |
| |
| /* Nothing to do. */ |
| return gcvSTATUS_OK; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Start |
| ** |
| ** Start up the command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to start. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Start( |
| IN gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| if (Command->feType == gcvHW_FE_WAIT_LINK) |
| { |
| gcmkONERROR(_StartWaitLinkFE(Command)); |
| } |
| else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) |
| { |
| gcmkONERROR(_StartMCFE(Command)); |
| } |
| else |
| { |
| gcmkONERROR(_StartAsyncFE(Command)); |
| } |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| |
| } |
| |
| static gceSTATUS |
| _StopWaitLinkFE( |
| IN gckCOMMAND Command |
| ) |
| { |
| gckHARDWARE hardware; |
| gceSTATUS status; |
| gctUINT32 idle; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Extract the gckHARDWARE object. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Replace last WAIT with END. */ |
| gcmkONERROR(gckWLFE_End( |
| hardware, |
| Command->waitPos.logical, |
| Command->waitPos.address, |
| &Command->waitPos.size |
| )); |
| |
| gcmkDUMP(Command->os, "#[end: fe stop]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_KERNEL_COMMAND, |
| Command->waitPos.logical, |
| Command->waitPos.address, |
| Command->waitPos.size |
| ); |
| |
| #if gcdSECURITY |
| gcmkONERROR(gckKERNEL_SecurityExecute( |
| Command->kernel, Command->waitPos.logical, 8 |
| )); |
| #endif |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail(Command->kernel->hardware, |
| Command->logical, |
| Command->offset)); |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->waitPos.videoMem, |
| Command->waitPos.offset, |
| Command->waitPos.logical, |
| Command->waitPos.size |
| )); |
| |
| /* Wait for idle. */ |
| gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle)); |
| |
| /* Command queue is no longer running. */ |
| Command->running = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _StopAsyncFE( |
| IN gckCOMMAND Command |
| ) |
| { |
| gckHARDWARE hardware; |
| gceSTATUS status; |
| gctUINT32 idle; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| hardware = Command->kernel->hardware; |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail(hardware, |
| Command->logical, |
| Command->offset)); |
| |
| /* Wait for idle. */ |
| gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle)); |
| |
| /* Command queue is no longer running. */ |
| Command->running = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _StopMCFE( |
| IN gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| hardware = Command->kernel->hardware; |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail(hardware, |
| Command->logical, |
| Command->offset)); |
| |
| /* Command queue is no longer running. */ |
| Command->running = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Stop |
| ** |
| ** Stop the command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object to stop. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Stop( |
| IN gckCOMMAND Command |
| ) |
| { |
| if (!Command->running) |
| { |
| /* Command queue is not running. */ |
| return gcvSTATUS_OK; |
| } |
| |
| if (Command->feType == gcvHW_FE_WAIT_LINK) |
| { |
| return _StopWaitLinkFE(Command); |
| } |
| else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) |
| { |
| return _StopMCFE(Command); |
| } |
| else |
| { |
| return _StopAsyncFE(Command); |
| } |
| } |
| |
| static gceSTATUS |
| _CommitWaitLinkOnce( |
| IN gckCOMMAND Command, |
| IN gckCONTEXT Context, |
| IN gcsHAL_COMMAND_LOCATION * CommandBuffer, |
| IN gcsSTATE_DELTA_PTR StateDelta, |
| IN gctUINT32 ProcessID, |
| IN gctBOOL Shared, |
| INOUT gctBOOL *contextSwitched |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL commitEntered = gcvFALSE; |
| gctBOOL contextAcquired = gcvFALSE; |
| gckHARDWARE hardware; |
| gcsPATCH_LIST_VARIABLE patchListVar = {0, 0}; |
| |
| gcsCONTEXT_PTR contextBuffer; |
| gctUINT8_PTR commandBufferLogical = gcvNULL; |
| gctUINT32 commandBufferAddress = 0; |
| gckVIDMEM_NODE commandBufferVideoMem = gcvNULL; |
| gctUINT8_PTR commandBufferTail = gcvNULL; |
| gctUINT commandBufferSize; |
| gctUINT32 linkBytes; |
| gctSIZE_T bytes; |
| gctUINT32 offset; |
| gctPOINTER entryLogical; |
| gctUINT32 entryAddress; |
| gctUINT32 entryBytes; |
| gctUINT32 exitAddress; |
| gctUINT32 exitBytes; |
| gctPOINTER waitLinkLogical; |
| gctUINT32 waitLinkAddress; |
| gctUINT32 waitLinkBytes; |
| gctUINT32 waitOffset; |
| gctUINT32 waitSize; |
| |
| #if gcdCAPTURE_ONLY_MODE |
| gctINT i; |
| #endif |
| |
| #ifdef __QNXNTO__ |
| gctPOINTER userCommandBufferLogical = gcvNULL; |
| gctBOOL userCommandBufferLogicalMapped = gcvFALSE; |
| #endif |
| |
| #if gcdDUMP_IN_KERNEL |
| gctPOINTER contextDumpLogical = gcvNULL; |
| # endif |
| gctUINT32 exitLinkLow = 0, exitLinkHigh = 0; |
| gctUINT32 entryLinkLow = 0, entryLinkHigh = 0; |
| gctUINT32 commandLinkLow = 0, commandLinkHigh = 0; |
| |
| gcmkHEADER_ARG("Command=%p CommandBuffer=%p ProcessID=%d", |
| Command, CommandBuffer, ProcessID |
| ); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| gcmkASSERT(Command->feType == gcvHW_FE_WAIT_LINK); |
| |
| /* Acquire the command queue. */ |
| gcmkONERROR(gckCOMMAND_EnterCommit(Command, gcvFALSE)); |
| commitEntered = gcvTRUE; |
| |
| /* Acquire the context switching mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContext, gcvINFINITE |
| )); |
| contextAcquired = gcvTRUE; |
| |
| /* Extract the gckHARDWARE and gckEVENT objects. */ |
| hardware = Command->kernel->hardware; |
| |
| |
| gcmkONERROR( |
| _HandlePatchList(Command, CommandBuffer, &patchListVar)); |
| |
| /* Query the size of LINK command. */ |
| gcmkONERROR(gckWLFE_Link( |
| hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL |
| )); |
| |
| /* Compute the command buffer entry and the size. */ |
| commandBufferLogical |
| = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) |
| + CommandBuffer->startOffset; |
| |
| commandBufferAddress = CommandBuffer->address |
| + CommandBuffer->startOffset; |
| |
| #ifdef __QNXNTO__ |
| gcmkONERROR(gckVIDMEM_HANDLE_Lookup( |
| Command->kernel, |
| ProcessID, |
| CommandBuffer->videoMemNode, |
| &commandBufferVideoMem |
| )); |
| |
| gcmkONERROR(gckVIDMEM_NODE_LockCPU( |
| Command->kernel, |
| commandBufferVideoMem, |
| gcvFALSE, |
| gcvFALSE, |
| &userCommandBufferLogical |
| )); |
| |
| commandBufferLogical = (gctUINT8_PTR)userCommandBufferLogical + CommandBuffer->startOffset; |
| userCommandBufferLogicalMapped =gcvTRUE; |
| #endif |
| |
| commandBufferSize = CommandBuffer->size; |
| |
| gcmkONERROR(_CheckFlushMMU(Command, hardware)); |
| |
| if (Command->dummyDraw == gcvTRUE && |
| Context != gcvNULL) |
| { |
| Command->dummyDraw = gcvFALSE; |
| gcmkONERROR(_DummyDraw(Command)); |
| } |
| |
| if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT) && |
| Command->kernel->asyncCommand && |
| patchListVar.maxAsyncTimestamp != 0) |
| { |
| gcmkONERROR(_WaitForAsyncCommandStamp( |
| Command, |
| patchListVar.maxAsyncTimestamp |
| )); |
| } |
| |
| /* Get the current offset. */ |
| offset = Command->offset; |
| |
| /* Compute number of bytes left in current kernel command queue. */ |
| bytes = Command->pageSize - offset; |
| |
| /* Query the size of WAIT/LINK command sequence. */ |
| gcmkONERROR(gckWLFE_WaitLink( |
| hardware, |
| gcvNULL, |
| ~0U, |
| offset, |
| &waitLinkBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| /* Is there enough space in the current command queue? */ |
| if (bytes < waitLinkBytes) |
| { |
| /* No, create a new one. */ |
| gcmkONERROR(_NewQueue(Command, gcvFALSE)); |
| |
| /* Get the new current offset. */ |
| offset = Command->offset; |
| |
| /* Recompute the number of bytes in the new kernel command queue. */ |
| bytes = Command->pageSize - offset; |
| gcmkASSERT(bytes >= waitLinkBytes); |
| } |
| |
| /* Compute the location if WAIT/LINK command sequence. */ |
| waitLinkLogical = (gctUINT8_PTR) Command->logical + offset; |
| waitLinkAddress = Command->address + offset; |
| |
| /* Context switch required? */ |
| if (Context == gcvNULL) |
| { |
| /* See if we have to switch pipes for the command buffer. */ |
| if (CommandBuffer->entryPipe == (gctUINT32)(Command->pipeSelect)) |
| { |
| /* Skip reserved head bytes. */ |
| offset = CommandBuffer->reservedHead; |
| } |
| else |
| { |
| gctUINT32 pipeBytes = CommandBuffer->reservedHead; |
| |
| /* The current hardware and the entry command buffer pipes |
| ** are different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| CommandBuffer->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| |
| /* Reserved bytes in userspace must be exact for a pipeSelect. */ |
| gcmkASSERT(pipeBytes == CommandBuffer->reservedHead); |
| } |
| |
| /* Compute the entry. */ |
| entryLogical = commandBufferLogical + offset; |
| entryAddress = commandBufferAddress + offset; |
| entryBytes = commandBufferSize - offset; |
| |
| Command->currContext = gcvNULL; |
| } |
| #if gcdDEBUG_OPTION && gcdDEBUG_FORCE_CONTEXT_UPDATE |
| else if (1) |
| #else |
| else if (Command->currContext != Context) |
| #endif |
| { |
| /* Get the current context buffer. */ |
| contextBuffer = Context->buffer; |
| |
| /* Yes, merge in the deltas. */ |
| gcmkONERROR(gckCONTEXT_Update(Context, ProcessID, StateDelta)); |
| |
| /*************************************************************** |
| ** SWITCHING CONTEXT. |
| */ |
| |
| /* Determine context buffer entry offset. */ |
| offset = (Command->pipeSelect == gcvPIPE_3D) |
| |
| /* Skip pipe switching sequence. */ |
| ? Context->entryOffset3D + Context->pipeSelectBytes |
| |
| /* Do not skip pipe switching sequence. */ |
| : Context->entryOffset3D; |
| |
| /* Compute the entry. */ |
| entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset; |
| entryAddress = contextBuffer->address + offset; |
| entryBytes = Context->bufferSize - offset; |
| |
| /* See if we have to switch pipes between the context |
| and command buffers. */ |
| if (CommandBuffer->entryPipe == gcvPIPE_3D) |
| { |
| /* Skip reserved head bytes. */ |
| offset = CommandBuffer->reservedHead; |
| } |
| else |
| { |
| gctUINT32 pipeBytes = CommandBuffer->reservedHead; |
| |
| /* The current hardware and the initial context pipes are |
| different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| CommandBuffer->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| |
| /* Reserved bytes in userspace must be exact for a pipeSelect. */ |
| gcmkASSERT(pipeBytes == CommandBuffer->reservedHead); |
| } |
| |
| /* Generate a LINK from the context buffer to |
| the command buffer. */ |
| gcmkONERROR(gckWLFE_Link( |
| hardware, |
| contextBuffer->link3D, |
| commandBufferAddress + offset, |
| commandBufferSize - offset, |
| &linkBytes, |
| &commandLinkLow, |
| &commandLinkHigh |
| )); |
| |
| #if gcdCAPTURE_ONLY_MODE |
| for (i = 0; i < gcdCONTEXT_BUFFER_NUM; ++i) |
| { |
| gcsCONTEXT_PTR buffer = contextBuffer; |
| |
| gckOS_CopyToUserData(Command->os, buffer->logical, CommandBuffer->contextLogical[i], Context->bufferSize); |
| |
| buffer = buffer->next; |
| } |
| #endif |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| contextBuffer->videoMem, |
| entryAddress - contextBuffer->address, |
| entryLogical, |
| entryBytes |
| )); |
| |
| /* Update the current context. */ |
| Command->currContext = Context; |
| |
| if (contextSwitched) |
| { |
| *contextSwitched = gcvTRUE; |
| } |
| |
| #if gcdDUMP_IN_KERNEL |
| contextDumpLogical = entryLogical; |
| #endif |
| |
| #if gcdSECURITY |
| /* Commit context buffer to trust zone. */ |
| gckKERNEL_SecurityExecute( |
| Command->kernel, |
| entryLogical, |
| entryBytes - 8 |
| ); |
| #endif |
| |
| #if gcdRECORD_COMMAND |
| gckRECORDER_Record( |
| Command->recorder, |
| gcvNULL, |
| 0xFFFFFFFF, |
| entryLogical, |
| entryBytes |
| ); |
| #endif |
| } |
| |
| /* Same context. */ |
| else |
| { |
| /* See if we have to switch pipes for the command buffer. */ |
| if (CommandBuffer->entryPipe == (gctUINT32)(Command->pipeSelect)) |
| { |
| /* Skip reserved head bytes. */ |
| offset = CommandBuffer->reservedHead; |
| } |
| else |
| { |
| gctUINT32 pipeBytes = CommandBuffer->reservedHead; |
| |
| /* The current hardware and the entry command buffer pipes |
| ** are different, switch to the correct pipe. */ |
| gcmkONERROR(gckHARDWARE_PipeSelect( |
| Command->kernel->hardware, |
| commandBufferLogical, |
| CommandBuffer->entryPipe, |
| &pipeBytes |
| )); |
| |
| /* Do not skip pipe switching sequence. */ |
| offset = 0; |
| |
| /* Reserved bytes in userspace must be exact for a pipeSelect. */ |
| gcmkASSERT(pipeBytes == CommandBuffer->reservedHead); |
| } |
| |
| /* Compute the entry. */ |
| entryLogical = commandBufferLogical + offset; |
| entryAddress = commandBufferAddress + offset; |
| entryBytes = commandBufferSize - offset; |
| } |
| |
| (void)entryLogical; |
| |
| /* Determine the location to jump to for the command buffer being |
| ** scheduled. */ |
| if (Command->newQueue) |
| { |
| /* New command queue, jump to the beginning of it. */ |
| /* Some extra commands (at beginning) are required for new queue. */ |
| exitAddress = Command->address; |
| exitBytes = Command->offset + waitLinkBytes; |
| } |
| else |
| { |
| /* Still within the preexisting command queue, jump to the new |
| WAIT/LINK command sequence. */ |
| exitAddress = waitLinkAddress; |
| exitBytes = waitLinkBytes; |
| } |
| |
| /* Add a new WAIT/LINK command sequence. When the command buffer which is |
| currently being scheduled is fully executed by the GPU, the FE will |
| jump to this WAIT/LINK sequence. */ |
| gcmkONERROR(gckWLFE_WaitLink( |
| hardware, |
| waitLinkLogical, |
| waitLinkAddress, |
| offset, |
| &waitLinkBytes, |
| &waitOffset, |
| &waitSize |
| )); |
| |
| if (Command->newQueue) |
| { |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->videoMem, |
| 0, |
| Command->logical, |
| exitBytes |
| )); |
| } |
| else |
| { |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->videoMem, |
| Command->offset, |
| waitLinkLogical, |
| exitBytes |
| )); |
| } |
| |
| /* Determine the location of the TAIL in the command buffer. */ |
| commandBufferTail |
| = commandBufferLogical |
| + commandBufferSize |
| - CommandBuffer->reservedTail; |
| |
| /* Generate command which writes out commit stamp. */ |
| if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT)) |
| { |
| gctUINT32 bytes; |
| |
| gcmkONERROR(gckHARDWARE_Fence( |
| hardware, |
| gcvENGINE_RENDER, |
| commandBufferTail, |
| Command->fence->address, |
| Command->commitStamp, |
| &bytes |
| )); |
| |
| commandBufferTail += bytes; |
| } |
| |
| /* Generate a LINK from the end of the command buffer being scheduled |
| back to the kernel command queue. */ |
| #if !gcdSECURITY |
| if (Shared == gcvFALSE) |
| { |
| gcmkONERROR(gckWLFE_Link( |
| hardware, |
| commandBufferTail, |
| exitAddress, |
| exitBytes, |
| &linkBytes, |
| &exitLinkLow, |
| &exitLinkHigh |
| )); |
| } |
| else |
| { |
| gctUINT8_PTR link = commandBufferTail + CommandBuffer->exitIndex * 16; |
| gctSIZE_T bytes = 8; |
| |
| gcmkONERROR(gckWLFE_ChipEnable( |
| hardware, |
| link, |
| (gceCORE_3D_MASK)(1 << hardware->kernel->chipID), |
| &bytes |
| )); |
| |
| link += bytes; |
| |
| gcmkONERROR(gckWLFE_Link( |
| hardware, |
| link, |
| exitAddress, |
| exitBytes, |
| &linkBytes, |
| &exitLinkLow, |
| &exitLinkHigh |
| )); |
| |
| link += linkBytes; |
| } |
| #endif |
| |
| gcmkONERROR(gckVIDMEM_HANDLE_Lookup( |
| Command->kernel, |
| ProcessID, |
| CommandBuffer->videoMemNode, |
| &commandBufferVideoMem |
| )); |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| commandBufferVideoMem, |
| CommandBuffer->startOffset, |
| commandBufferLogical, |
| commandBufferSize |
| )); |
| |
| #if gcdRECORD_COMMAND |
| gckRECORDER_Record( |
| Command->recorder, |
| commandBufferLogical + offset, |
| commandBufferSize - offset, |
| gcvNULL, |
| 0xFFFFFFFF |
| ); |
| |
| gckRECORDER_AdvanceIndex(Command->recorder, Command->commitStamp); |
| #endif |
| |
| #if gcdSECURITY |
| /* Submit command buffer to trust zone. */ |
| gckKERNEL_SecurityExecute( |
| Command->kernel, |
| commandBufferLogical + offset, |
| commandBufferSize - offset - 8 |
| ); |
| #else |
| #if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE |
| /* |
| * Skip link to entryAddress. |
| * Instead, we directly link to final wait link position. |
| */ |
| gcmkONERROR(gckWLFE_Link( |
| hardware, |
| Command->waitPos.logical, |
| waitLinkAddress, |
| waitLinkBytes, |
| &Command->waitPos.size, |
| &entryLinkLow, |
| &entryLinkHigh |
| )); |
| # else |
| /* Generate a LINK from the previous WAIT/LINK command sequence to the |
| entry determined above (either the context or the command buffer). |
| This LINK replaces the WAIT instruction from the previous WAIT/LINK |
| pair, therefore we use WAIT metrics for generation of this LINK. |
| This action will execute the entire sequence. */ |
| gcmkONERROR(gckWLFE_Link( |
| hardware, |
| Command->waitPos.logical, |
| entryAddress, |
| entryBytes, |
| &Command->waitPos.size, |
| &entryLinkLow, |
| &entryLinkHigh |
| )); |
| # endif |
| #endif |
| |
| #if gcdLINK_QUEUE_SIZE |
| /* TODO: What's it? */ |
| if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_USER_COMMAND) |
| { |
| gcuQUEUEDATA data; |
| |
| gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); |
| |
| data.linkData.start = entryAddress; |
| data.linkData.end = entryAddress + entryBytes; |
| data.linkData.linkLow = entryLinkLow; |
| data.linkData.linkHigh = entryLinkHigh; |
| |
| gckQUEUE_Enqueue(&hardware->linkQueue, &data); |
| |
| if (commandBufferAddress + offset != entryAddress) |
| { |
| data.linkData.start = commandBufferAddress + offset; |
| data.linkData.end = commandBufferAddress + commandBufferSize; |
| data.linkData.linkLow = commandLinkLow; |
| data.linkData.linkHigh = commandLinkHigh; |
| |
| gckQUEUE_Enqueue(&hardware->linkQueue, &data); |
| } |
| |
| if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) |
| { |
| data.linkData.start = exitAddress; |
| data.linkData.end = exitAddress + exitBytes; |
| data.linkData.linkLow = exitLinkLow; |
| data.linkData.linkHigh = exitLinkHigh; |
| |
| /* Dump kernel command.*/ |
| gckQUEUE_Enqueue(&hardware->linkQueue, &data); |
| } |
| } |
| #endif |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->waitPos.videoMem, |
| Command->waitPos.offset, |
| Command->waitPos.logical, |
| Command->waitPos.size |
| )); |
| |
| if (entryAddress != commandBufferAddress + offset) |
| { |
| gcmkDUMP(Command->os, "#[context]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_KERNEL_CONTEXT, |
| contextDumpLogical, |
| entryAddress, |
| entryBytes |
| ); |
| |
| /* execute context. */ |
| gcmkDUMP(Command->os, |
| "@[execute 0 0 0x%08X 0x%08X]", |
| entryAddress + offset, |
| entryBytes - offset - 8 |
| ); |
| } |
| |
| gcmkDUMP(Command->os, "#[command: user]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_COMMAND, |
| commandBufferLogical + offset, |
| commandBufferAddress + offset, |
| commandBufferSize - offset |
| ); |
| |
| /* execute user commands. */ |
| gcmkDUMP( |
| Command->os, |
| "@[execute 0 0 0x%08X 0x%08X]", |
| commandBufferAddress |
| + CommandBuffer->reservedHead, |
| commandBufferSize |
| - CommandBuffer->reservedHead |
| - CommandBuffer->reservedTail |
| ); |
| |
| gcmkDUMP(Command->os, "#[wait-link]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_KERNEL_COMMAND, |
| waitLinkLogical, |
| waitLinkAddress, |
| waitLinkBytes |
| ); |
| |
| #if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE |
| gcmkDUMP( |
| Command->os, |
| "#[null driver: below command skipped link to 0x%08X 0x%08X]", |
| entryAddress, |
| entryBytes |
| ); |
| #endif |
| |
| gcmkDUMP(Command->os, "#[link: break prev wait-link]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_KERNEL_COMMAND, |
| Command->waitPos.logical, |
| Command->waitPos.address, |
| Command->waitPos.size |
| ); |
| |
| /* Update the current pipe. */ |
| Command->pipeSelect = CommandBuffer->exitPipe; |
| |
| /* Update command queue offset. */ |
| Command->offset += waitLinkBytes; |
| Command->newQueue = gcvFALSE; |
| |
| /* Update address of last WAIT. */ |
| Command->waitPos.videoMem = Command->videoMem; |
| Command->waitPos.offset = Command->offset - waitLinkBytes + waitOffset; |
| Command->waitPos.logical = (gctUINT8_PTR)waitLinkLogical + waitOffset; |
| Command->waitPos.address = waitLinkAddress + waitOffset; |
| Command->waitPos.size = waitSize; |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail( |
| hardware, Command->logical, Command->offset |
| )); |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| contextAcquired = gcvFALSE; |
| |
| /* Release the command queue. */ |
| gcmkONERROR(gckCOMMAND_ExitCommit(Command, gcvFALSE)); |
| commitEntered = gcvFALSE; |
| |
| if (status == gcvSTATUS_INTERRUPTED) |
| { |
| gcmkTRACE( |
| gcvLEVEL_INFO, |
| "%s(%d): Intterupted in gckEVENT_Submit", |
| __FUNCTION__, __LINE__ |
| ); |
| status = gcvSTATUS_OK; |
| } |
| else |
| { |
| gcmkONERROR(status); |
| } |
| |
| #ifdef __QNXNTO__ |
| if(userCommandBufferLogicalMapped) |
| { |
| gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( |
| Command->kernel, |
| commandBufferVideoMem, |
| ProcessID, |
| gcvFALSE, |
| gcvFALSE |
| )); |
| |
| userCommandBufferLogicalMapped =gcvFALSE; |
| } |
| #endif |
| |
| /* Return status. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (contextAcquired) |
| { |
| /* Release the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| } |
| |
| if (commitEntered) |
| { |
| /* Release the command queue mutex. */ |
| gcmkVERIFY_OK(gckCOMMAND_ExitCommit(Command, gcvFALSE)); |
| } |
| |
| #ifdef __QNXNTO__ |
| if (userCommandBufferLogicalMapped) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| userCommandBufferLogical, |
| 0, |
| commandBufferLogical)); |
| } |
| #endif |
| |
| /* Return status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _CommitAsyncOnce( |
| IN gckCOMMAND Command, |
| IN gcsHAL_COMMAND_LOCATION * CommandBuffer |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware = Command->kernel->hardware; |
| gctBOOL available = gcvFALSE; |
| gctBOOL acquired = gcvFALSE; |
| gctUINT8_PTR commandBufferLogical; |
| gctUINT8_PTR commandBufferTail; |
| gctUINT commandBufferSize; |
| gctUINT32 commandBufferAddress; |
| gctUINT32 fenceBytes; |
| gctUINT32 oldValue; |
| gctUINT32 flushBytes; |
| gcsPATCH_LIST_VARIABLE patchListVar = {0, 0}; |
| |
| gcmkHEADER(); |
| |
| gcmkVERIFY_OK( |
| _HandlePatchList(Command, CommandBuffer, &patchListVar)); |
| |
| gckOS_AtomicExchange(Command->os, |
| hardware->pageTableDirty[gcvENGINE_BLT], |
| 0, |
| &oldValue); |
| |
| if (oldValue) |
| { |
| gckHARDWARE_FlushAsyncMMU(hardware, gcvNULL, &flushBytes); |
| |
| gcmkASSERT(flushBytes <= CommandBuffer->reservedHead); |
| |
| /* Compute the command buffer entry to insert the flushMMU commands. */ |
| commandBufferLogical = (gctUINT8_PTR)gcmUINT64_TO_PTR(CommandBuffer->logical) |
| + CommandBuffer->startOffset |
| + CommandBuffer->reservedHead |
| - flushBytes; |
| |
| commandBufferAddress = CommandBuffer->address |
| + CommandBuffer->startOffset |
| + CommandBuffer->reservedHead |
| - flushBytes; |
| |
| gckHARDWARE_FlushAsyncMMU(hardware, commandBufferLogical, &flushBytes); |
| } |
| else |
| { |
| /* Compute the command buffer entry. */ |
| commandBufferLogical = (gctUINT8_PTR)gcmUINT64_TO_PTR(CommandBuffer->logical) |
| + CommandBuffer->startOffset |
| + CommandBuffer->reservedHead; |
| |
| commandBufferAddress = CommandBuffer->address |
| + CommandBuffer->startOffset |
| + CommandBuffer->reservedHead; |
| |
| flushBytes = 0; |
| } |
| |
| commandBufferTail = (gctUINT8_PTR)gcmUINT64_TO_PTR(CommandBuffer->logical) |
| + CommandBuffer->startOffset |
| + CommandBuffer->size |
| - CommandBuffer->reservedTail; |
| |
| gcmkONERROR(gckHARDWARE_Fence( |
| hardware, |
| gcvENGINE_BLT, |
| commandBufferTail, |
| Command->fence->address, |
| Command->commitStamp, |
| &fenceBytes |
| )); |
| |
| gcmkASSERT(fenceBytes <= CommandBuffer->reservedTail); |
| |
| commandBufferSize = CommandBuffer->size |
| - CommandBuffer->reservedHead |
| - CommandBuffer->reservedTail |
| + flushBytes |
| + fenceBytes; |
| |
| gckOS_AcquireMutex(Command->os, Command->mutexContext, gcvINFINITE); |
| acquired = gcvTRUE; |
| |
| /* Acquire a slot. */ |
| for (;;) |
| { |
| gcmkONERROR(gckASYNC_FE_ReserveSlot(hardware, &available)); |
| |
| if (available) |
| { |
| break; |
| } |
| else |
| { |
| gcmkTRACE_ZONE(gcvLEVEL_INFO, _GC_OBJ_ZONE, "No available slot, have to wait"); |
| |
| gckOS_Delay(Command->os, 1); |
| } |
| } |
| |
| #if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE |
| /* Skip submit to hardware for NULL driver. */ |
| gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); |
| #endif |
| |
| gcmkDUMP(Command->os, "#[async-command: user]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_ASYNC_COMMAND, |
| commandBufferLogical, |
| commandBufferAddress, |
| commandBufferSize |
| ); |
| |
| gcmkDUMP( |
| Command->os, |
| "@[execute 1 0 0x%08X 0x%08X]", |
| commandBufferAddress |
| + CommandBuffer->reservedHead, |
| CommandBuffer->size |
| - CommandBuffer->reservedHead |
| - CommandBuffer->reservedTail |
| ); |
| |
| #if !gcdNULL_DRIVER |
| /* Execute command buffer. */ |
| gckASYNC_FE_Execute(hardware, commandBufferAddress, commandBufferSize); |
| #endif |
| |
| gckOS_ReleaseMutex(Command->os, Command->mutexContext); |
| acquired = gcvFALSE; |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| gckOS_ReleaseMutex(Command->os, Command->mutexContext); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| static gceSTATUS |
| _CommitMultiChannelOnce( |
| IN gckCOMMAND Command, |
| IN gckCONTEXT Context, |
| IN gcsHAL_COMMAND_LOCATION * CommandBuffer |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL acquired = gcvFALSE; |
| gctUINT8_PTR commandBufferLogical; |
| gctUINT commandBufferSize; |
| gctUINT32 commandBufferAddress; |
| gckHARDWARE hardware; |
| gctUINT64 bit; |
| gcsPATCH_LIST_VARIABLE patchListVar = {0, 0}; |
| |
| gcmkHEADER_ARG("priority=%d channelId=%d videoMemNode=%u size=0x%x patchHead=%p", |
| CommandBuffer->priority, CommandBuffer->channelId, |
| CommandBuffer->videoMemNode, CommandBuffer->size, |
| gcmUINT64_TO_PTR(CommandBuffer->patchHead)); |
| |
| gcmkASSERT(Command->feType == gcvHW_FE_MULTI_CHANNEL); |
| |
| hardware = Command->kernel->hardware; |
| |
| gcmkVERIFY_OK( |
| _HandlePatchList(Command, CommandBuffer, &patchListVar)); |
| |
| /* Check flush mcfe MMU cache. */ |
| gcmkONERROR(_CheckFlushMcfeMMU(Command, hardware)); |
| |
| /* Compute the command buffer entry and the size. */ |
| commandBufferLogical |
| = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) |
| + CommandBuffer->startOffset |
| + CommandBuffer->reservedHead; |
| |
| commandBufferAddress = CommandBuffer->address |
| + CommandBuffer->startOffset |
| + CommandBuffer->reservedHead; |
| |
| /* reservedTail bytes are not used, because fence not enable. */ |
| commandBufferSize |
| = CommandBuffer->size |
| - CommandBuffer->reservedHead |
| - CommandBuffer->reservedTail; |
| |
| if (commandBufferSize & 8) |
| { |
| /* |
| * Need 16 byte alignment for MCFE command size. |
| * command is already 8 byte aligned, if not 16 byte aligned, |
| * we need append 8 bytes. |
| */ |
| gctUINT32 nop[2]; |
| gctSIZE_T bytes = 8; |
| gctUINT8_PTR tail = commandBufferLogical + commandBufferSize; |
| |
| gckMCFE_Nop(hardware, nop, &bytes); |
| gcmkASSERT(bytes == 8); |
| |
| gckOS_WriteMemory(Command->os, tail, nop[0]); |
| gckOS_WriteMemory(Command->os, tail + 4, nop[1]); |
| |
| commandBufferSize += 8; |
| } |
| |
| /* Large command buffer size does not make sense. */ |
| gcmkASSERT(commandBufferSize < 0x800000); |
| |
| |
| if (CommandBuffer->channelId != 0) |
| { |
| /* Sync from the system channel. */ |
| gcmkONERROR(_SyncFromSystemChannel( |
| Command, |
| (gctBOOL)CommandBuffer->priority, |
| (gctUINT32)CommandBuffer->channelId |
| )); |
| } |
| |
| Command->currContext = Context; |
| |
| gckOS_AcquireMutex(Command->os, Command->mutexQueue, gcvINFINITE); |
| acquired = gcvTRUE; |
| |
| #if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE |
| /* Skip submit to hardware for NULL driver. */ |
| gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); |
| #endif |
| |
| gcmkDUMP(Command->os, "#[mcfe-command: user]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_COMMAND, |
| commandBufferLogical, |
| commandBufferAddress, |
| commandBufferSize |
| ); |
| |
| gcmkDUMP(Command->os, |
| "@[execute %d %d 0x%08X 0x%08X]", |
| CommandBuffer->channelId, |
| CommandBuffer->priority, |
| commandBufferAddress, |
| commandBufferSize); |
| |
| #if !gcdNULL_DRIVER |
| /* Execute command buffer. */ |
| gcmkONERROR(gckMCFE_Execute( |
| hardware, |
| (gctBOOL)CommandBuffer->priority, |
| (gctUINT32)CommandBuffer->channelId, |
| commandBufferAddress, |
| commandBufferSize |
| )); |
| #endif |
| |
| bit = 1ull << CommandBuffer->channelId; |
| |
| /* This channel is dirty. */ |
| Command->dirtyChannel[CommandBuffer->priority ? 1 : 0] |= bit; |
| |
| gckOS_ReleaseMutex(Command->os, Command->mutexQueue); |
| acquired = gcvFALSE; |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (acquired) |
| { |
| gckOS_ReleaseMutex(Command->os, Command->mutexQueue); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Commit |
| ** |
| ** Commit command buffers to the command queue. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** gcsHAL_SUBCOMMIT * SubCommit |
| ** Commit information, includes context, delta, and command buffer |
| * locations. |
| ** |
| ** gctUINT32 ProcessID |
| ** Current process ID. |
| ** |
| ** OUTPUT: |
| ** gctBOOL *contextSwitched |
| ** pass context Switch flag to upper |
| */ |
| gceSTATUS |
| gckCOMMAND_Commit( |
| IN gckCOMMAND Command, |
| IN gcsHAL_SUBCOMMIT * SubCommit, |
| IN gctUINT32 ProcessId, |
| IN gctBOOL Shared, |
| OUT gctUINT64_PTR CommitStamp, |
| INOUT gctBOOL *contextSwitched |
| ) |
| { |
| gceSTATUS status; |
| gcsSTATE_DELTA_PTR delta = gcmUINT64_TO_PTR(SubCommit->delta); |
| gckCONTEXT context = gcvNULL; |
| gcsHAL_COMMAND_LOCATION *cmdLoc = &SubCommit->commandBuffer; |
| gcsHAL_COMMAND_LOCATION _cmdLoc; |
| gctPOINTER userPtr = gcvNULL; |
| gctBOOL needCopy = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=%p SubCommit=%p delta=%p context=%u pid=%u", |
| Command, SubCommit, delta, SubCommit->context, ProcessId); |
| |
| gcmkVERIFY_OK(gckOS_QueryNeedCopy(Command->os, ProcessId, &needCopy)); |
| |
| if (SubCommit->context) |
| { |
| context = gckKERNEL_QueryPointerFromName( |
| Command->kernel, |
| (gctUINT32)(SubCommit->context) |
| ); |
| } |
| |
| do |
| { |
| gctUINT64 next; |
| |
| /* Skip the first nested one. */ |
| if (userPtr) |
| { |
| /* Copy/map command buffer location from user. */ |
| if (needCopy) |
| { |
| cmdLoc = &_cmdLoc; |
| |
| status = gckOS_CopyFromUserData( |
| Command->os, |
| cmdLoc, |
| userPtr, |
| gcmSIZEOF(gcsHAL_COMMAND_LOCATION) |
| ); |
| } |
| else |
| { |
| status = gckOS_MapUserPointer( |
| Command->os, |
| userPtr, |
| gcmSIZEOF(gcsHAL_COMMAND_LOCATION), |
| (gctPOINTER *)&cmdLoc |
| ); |
| } |
| |
| if (gcmIS_ERROR(status)) |
| { |
| userPtr = gcvNULL; |
| |
| gcmkONERROR(status); |
| } |
| } |
| |
| if (Command->feType == gcvHW_FE_WAIT_LINK) |
| { |
| /* Commit command buffers. */ |
| status = _CommitWaitLinkOnce(Command, |
| context, |
| cmdLoc, |
| delta, |
| ProcessId, |
| Shared, |
| contextSwitched); |
| } |
| else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) |
| { |
| status = _CommitMultiChannelOnce(Command, context, cmdLoc); |
| } |
| else |
| { |
| gcmkASSERT(Command->feType == gcvHW_FE_ASYNC); |
| |
| status = _CommitAsyncOnce(Command, cmdLoc); |
| } |
| |
| if (status != gcvSTATUS_INTERRUPTED) |
| { |
| gcmkONERROR(status); |
| } |
| |
| /* Do not need context or delta for later commands. */ |
| context = gcvNULL; |
| delta = gcvNULL; |
| |
| next = cmdLoc->next; |
| |
| /* Unmap user pointer if mapped. */ |
| if (!needCopy && userPtr) |
| { |
| gcmkVERIFY_OK(gckOS_UnmapUserPointer( |
| Command->os, |
| userPtr, |
| gcmSIZEOF(gcsHAL_COMMAND_LOCATION), |
| cmdLoc |
| )); |
| } |
| |
| /* Advance to next command buffer location from user. */ |
| userPtr = gcmUINT64_TO_PTR(next); |
| } |
| while (userPtr); |
| |
| if (Command->feType == gcvHW_FE_MULTI_CHANNEL) |
| { |
| /* |
| * Semphore synchronization. |
| * |
| * Here we blindly sync dirty other channels to the system channel. |
| * The scenario to sync channels to the system channel: |
| * 1. Need to sync channels who sent semaphores. |
| * 2. Need to sync dirty channels when event(interrupt) is to sent. |
| * 3. Need to sync dirty channels when system channel need run something |
| * such as flush mmu. |
| * |
| * When power management is on, blindly sync dirty channels is OK because |
| * there's always a event(intrrupt). |
| * |
| * The only condition we sync more than needed is: |
| * a. power manangement is off. |
| * b. no user event is attached when commit. |
| * c. no user event is to be submitted in next ioctl. |
| * That's a rare condition. |
| * |
| * Conclusion is that, blindly sync dirty channels is a good choice for |
| * now. |
| */ |
| gcmkONERROR(_SyncToSystemChannel(Command, Command->dirtyChannel)); |
| } |
| |
| /* Output commit stamp. */ |
| *CommitStamp = Command->commitStamp; |
| Command->commitStamp++; |
| |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (!needCopy && userPtr) |
| { |
| gckOS_UnmapUserPointer( |
| Command->os, |
| userPtr, |
| gcmSIZEOF(gcsHAL_COMMAND_LOCATION), |
| cmdLoc |
| ); |
| } |
| |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Reserve |
| ** |
| ** Reserve space in the command queue. Also acquire the command queue mutex. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctSIZE_T RequestedBytes |
| ** Number of bytes previously reserved. |
| ** |
| ** OUTPUT: |
| ** |
| ** gctPOINTER * Buffer |
| ** Pointer to a variable that will receive the address of the reserved |
| ** space. |
| ** |
| ** gctSIZE_T * BufferSize |
| ** Pointer to a variable that will receive the number of bytes |
| ** available in the command queue. |
| */ |
| gceSTATUS |
| gckCOMMAND_Reserve( |
| IN gckCOMMAND Command, |
| IN gctUINT32 RequestedBytes, |
| OUT gctPOINTER * Buffer, |
| OUT gctUINT32 * BufferSize |
| ) |
| { |
| gceSTATUS status; |
| gctUINT32 bytes; |
| gctUINT32 requiredBytes; |
| gctUINT32 requestedAligned; |
| |
| gcmkHEADER_ARG("Command=%p RequestedBytes=%lu", Command, RequestedBytes); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| if (Command->feType == gcvHW_FE_WAIT_LINK) |
| { |
| /* Compute aligned number of reuested bytes. */ |
| requestedAligned = gcmALIGN(RequestedBytes, Command->alignment); |
| |
| /* Another WAIT/LINK command sequence will have to be appended after |
| the requested area being reserved. Compute the number of bytes |
| required for WAIT/LINK at the location after the reserved area. */ |
| gcmkONERROR(gckWLFE_WaitLink( |
| Command->kernel->hardware, |
| gcvNULL, |
| ~0U, |
| Command->offset + requestedAligned, |
| &requiredBytes, |
| gcvNULL, |
| gcvNULL |
| )); |
| |
| /* Compute total number of bytes required. */ |
| requiredBytes += requestedAligned; |
| } |
| else |
| { |
| requiredBytes = gcmALIGN(RequestedBytes, 8); |
| } |
| |
| /* Compute number of bytes available in command queue. */ |
| bytes = Command->pageSize - Command->offset; |
| |
| /* Is there enough space in the current command queue? */ |
| if (bytes < requiredBytes) |
| { |
| /* Create a new command queue. */ |
| gcmkONERROR(_NewQueue(Command, gcvFALSE)); |
| |
| /* Recompute the number of bytes in the new kernel command queue. */ |
| bytes = Command->pageSize - Command->offset; |
| |
| /* Still not enough space? */ |
| if (bytes < requiredBytes) |
| { |
| /* Rare case, not enough room in command queue. */ |
| gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); |
| } |
| } |
| |
| /* Return pointer to empty slot command queue. */ |
| *Buffer = (gctUINT8 *) Command->logical + Command->offset; |
| |
| /* Return number of bytes left in command queue. */ |
| *BufferSize = bytes; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Buffer=0x%x *BufferSize=%lu", *Buffer, *BufferSize); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Execute |
| ** |
| ** Execute a previously reserved command queue by appending a WAIT/LINK command |
| ** sequence after it and modifying the last WAIT into a LINK command. The |
| ** command FIFO mutex will be released whether this function succeeds or not. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctSIZE_T RequestedBytes |
| ** Number of bytes previously reserved. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Execute( |
| IN gckCOMMAND Command, |
| IN gctUINT32 RequestedBytes |
| ) |
| { |
| gceSTATUS status; |
| |
| gctUINT8_PTR waitLinkLogical; |
| gctUINT32 waitLinkAddress; |
| gctUINT32 waitLinkOffset; |
| gctUINT32 waitLinkBytes; |
| |
| gctUINT32 waitOffset; |
| gctUINT32 waitBytes; |
| |
| gctUINT32 linkLow, linkHigh; |
| |
| gctPOINTER execLogical; |
| gctUINT32 execAddress; |
| gctUINT32 execBytes; |
| |
| gcmkHEADER_ARG("Command=%p RequestedBytes=%lu", Command, RequestedBytes); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Compute offset for WAIT/LINK. */ |
| waitLinkOffset = Command->offset + RequestedBytes; |
| |
| /* Compute number of bytes left in command queue. */ |
| waitLinkBytes = Command->pageSize - waitLinkOffset; |
| |
| /* Compute the location if WAIT/LINK command sequence. */ |
| waitLinkLogical = (gctUINT8_PTR) Command->logical + waitLinkOffset; |
| waitLinkAddress = Command->address + waitLinkOffset; |
| |
| /* Append WAIT/LINK in command queue. */ |
| gcmkONERROR(gckWLFE_WaitLink( |
| Command->kernel->hardware, |
| waitLinkLogical, |
| waitLinkAddress, |
| waitLinkOffset, |
| &waitLinkBytes, |
| &waitOffset, |
| &waitBytes |
| )); |
| |
| |
| /* Determine the location to jump to for the command buffer being |
| ** scheduled. */ |
| if (Command->newQueue) |
| { |
| /* New command queue, jump to the beginning of it. */ |
| execLogical = Command->logical; |
| execAddress = Command->address; |
| execBytes = Command->offset + RequestedBytes + waitLinkBytes; |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->videoMem, |
| 0, |
| execLogical, |
| execBytes |
| )); |
| } |
| else |
| { |
| /* Still within the preexisting command queue, jump directly to the |
| reserved area. */ |
| execLogical = (gctUINT8 *) Command->logical + Command->offset; |
| execAddress = Command->address + Command->offset; |
| execBytes = RequestedBytes + waitLinkBytes; |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->videoMem, |
| Command->offset, |
| execLogical, |
| execBytes |
| )); |
| } |
| |
| #if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE |
| /* |
| * Skip link to execAddress. |
| * Instead, we directly link to final wait link position. |
| */ |
| gcmkONERROR(gckWLFE_Link( |
| Command->kernel->hardware, |
| Command->waitPos.logical, |
| waitLinkAddress, |
| waitLinkBytes, |
| &Command->waitPos.size, |
| &linkLow, |
| &linkHigh |
| )); |
| #else |
| /* Convert the last WAIT into a LINK. */ |
| gcmkONERROR(gckWLFE_Link( |
| Command->kernel->hardware, |
| Command->waitPos.logical, |
| execAddress, |
| execBytes, |
| &Command->waitPos.size, |
| &linkLow, |
| &linkHigh |
| )); |
| #endif |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->waitPos.videoMem, |
| Command->waitPos.offset, |
| Command->waitPos.logical, |
| Command->waitPos.size |
| )); |
| |
| #if gcdLINK_QUEUE_SIZE |
| if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) |
| { |
| gcuQUEUEDATA data; |
| |
| gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); |
| |
| data.linkData.start = execAddress; |
| data.linkData.end = execAddress + execBytes; |
| data.linkData.linkLow = linkLow; |
| data.linkData.linkHigh = linkHigh; |
| |
| gckQUEUE_Enqueue(&Command->kernel->hardware->linkQueue, &data); |
| } |
| #endif |
| |
| gcmkDUMP(Command->os, "#[command: kernel execute]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_KERNEL_COMMAND, |
| execLogical, |
| execAddress, |
| execBytes |
| ); |
| |
| #if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE |
| gcmkDUMP(Command->os, |
| "#[null driver: below command skipped link to 0x%08X 0x%08X]", |
| execAddress, |
| execBytes); |
| #endif |
| |
| gcmkDUMP(Command->os, "#[link: break prev wait-link]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_KERNEL_COMMAND, |
| Command->waitPos.logical, |
| Command->waitPos.address, |
| Command->waitPos.size |
| ); |
| |
| /* Update the pointer to the last WAIT. */ |
| Command->waitPos.videoMem = Command->videoMem; |
| Command->waitPos.offset = waitLinkOffset + waitOffset; |
| Command->waitPos.logical = (gctUINT8_PTR)waitLinkLogical + waitOffset; |
| Command->waitPos.address = waitLinkAddress + waitOffset; |
| Command->waitPos.size = waitBytes; |
| |
| /* Update the command queue. */ |
| Command->offset += RequestedBytes + waitLinkBytes; |
| Command->newQueue = gcvFALSE; |
| |
| /* Update queue tail pointer. */ |
| gcmkONERROR(gckHARDWARE_UpdateQueueTail( |
| Command->kernel->hardware, Command->logical, Command->offset |
| )); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| gceSTATUS |
| gckCOMMAND_ExecuteAsync( |
| IN gckCOMMAND Command, |
| IN gctUINT32 RequestedBytes |
| ) |
| { |
| gceSTATUS status; |
| gckHARDWARE hardware; |
| gctBOOL available; |
| gctPOINTER execLogical; |
| gctUINT32 execAddress; |
| gctUINT32 execBytes; |
| |
| hardware = Command->kernel->hardware; |
| |
| /* Determine the location to jump to for the command buffer being |
| ** scheduled. */ |
| if (Command->newQueue) |
| { |
| /* New command queue, jump to the beginning of it. */ |
| execLogical = Command->logical; |
| execAddress = Command->address; |
| execBytes = Command->offset + RequestedBytes; |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->videoMem, |
| 0, |
| execLogical, |
| execBytes |
| )); |
| } |
| else |
| { |
| /* Still within the preexisting command queue, jump directly to the |
| reserved area. */ |
| execLogical = (gctUINT8 *) Command->logical + Command->offset; |
| execAddress = Command->address + Command->offset; |
| execBytes = RequestedBytes; |
| |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->videoMem, |
| Command->offset, |
| execLogical, |
| execBytes |
| )); |
| } |
| |
| /* Acquire a slot. */ |
| for (;;) |
| { |
| gcmkONERROR(gckASYNC_FE_ReserveSlot(hardware, &available)); |
| |
| if (available) |
| { |
| break; |
| } |
| else |
| { |
| gckOS_Delay(Command->os, 1); |
| } |
| } |
| |
| #if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE |
| /* Skip submit to hardware for NULL driver. */ |
| gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); |
| #endif |
| |
| gcmkDUMP(Command->os, "#[async-command: kernel execute]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_KERNEL_COMMAND, |
| execLogical, |
| execAddress, |
| execBytes |
| ); |
| |
| #if !gcdNULL_DRIVER |
| /* Send descriptor. */ |
| gckASYNC_FE_Execute(hardware, execAddress, execBytes); |
| #endif |
| |
| /* Update the command queue. */ |
| Command->offset += RequestedBytes; |
| Command->newQueue = gcvFALSE; |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| return status; |
| } |
| |
| gceSTATUS |
| gckCOMMAND_ExecuteMultiChannel( |
| IN gckCOMMAND Command, |
| IN gctBOOL Priority, |
| IN gctUINT32 ChannelId, |
| IN gctUINT32 RequestedBytes |
| ) |
| { |
| gceSTATUS status; |
| gctPOINTER execLogical; |
| gctUINT32 execAddress; |
| gctUINT32 execBytes; |
| |
| /* Determine the location to jump to for the command buffer being |
| ** scheduled. */ |
| if (Command->newQueue) |
| { |
| /* New command queue, jump to the beginning of it. */ |
| execLogical = Command->logical; |
| execAddress = Command->address; |
| execBytes = Command->offset + RequestedBytes; |
| } |
| else |
| { |
| /* Still within the preexisting command queue, jump directly to the |
| reserved area. */ |
| execLogical = (gctUINT8 *) Command->logical + Command->offset; |
| execAddress = Command->address + Command->offset; |
| execBytes = RequestedBytes; |
| } |
| |
| if (execBytes & 8) |
| { |
| gctSIZE_T bytes = 8; |
| gctUINT8_PTR tail = (gctUINT8_PTR)execLogical + execBytes; |
| |
| gckMCFE_Nop(Command->kernel->hardware, tail, &bytes); |
| gcmkASSERT(bytes == 8); |
| |
| execBytes += 8; |
| RequestedBytes += 8; |
| } |
| |
| if (Command->newQueue) |
| { |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->videoMem, |
| 0, |
| execLogical, |
| execBytes |
| )); |
| } |
| else |
| { |
| gcmkONERROR(gckVIDMEM_NODE_CleanCache( |
| Command->kernel, |
| Command->videoMem, |
| Command->offset, |
| execLogical, |
| execBytes |
| )); |
| } |
| |
| #if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE |
| /* Skip submit to hardware for NULL driver. */ |
| gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); |
| #endif |
| |
| gcmkDUMP(Command->os, "#[mcfe-command: kernel execute]"); |
| gcmkDUMP_BUFFER( |
| Command->os, |
| gcvDUMP_BUFFER_KERNEL_COMMAND, |
| execLogical, |
| execAddress, |
| execBytes |
| ); |
| |
| gcmkDUMP(Command->os, |
| "@[execute %u %u 0x%08X 0x%08X]", |
| ChannelId, Priority, execAddress, execBytes); |
| |
| #if !gcdNULL_DRIVER |
| /* Send descriptor. */ |
| gcmkONERROR( |
| gckMCFE_Execute(Command->kernel->hardware, |
| Priority, |
| ChannelId, |
| execAddress, |
| execBytes)); |
| #endif |
| |
| /* Update the command queue. */ |
| Command->offset += RequestedBytes; |
| Command->newQueue = gcvFALSE; |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Stall |
| ** |
| ** The calling thread will be suspended until the command queue has been |
| ** completed. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to an gckCOMMAND object. |
| ** |
| ** gctBOOL FromPower |
| ** Determines whether the call originates from inside the power |
| ** management or not. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Stall( |
| IN gckCOMMAND Command, |
| IN gctBOOL FromPower |
| ) |
| { |
| gckOS os; |
| gckHARDWARE hardware; |
| gckEVENT eventObject; |
| gceSTATUS status; |
| gctSIGNAL signal = gcvNULL; |
| gctUINT timer = 0; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Extract the gckOS object pointer. */ |
| os = Command->os; |
| gcmkVERIFY_OBJECT(os, gcvOBJ_OS); |
| |
| /* Extract the gckHARDWARE object pointer. */ |
| hardware = Command->kernel->hardware; |
| gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); |
| |
| /* Extract the gckEVENT object pointer. */ |
| eventObject = Command->kernel->eventObj; |
| gcmkVERIFY_OBJECT(eventObject, gcvOBJ_EVENT); |
| |
| /* Allocate the signal. */ |
| gcmkONERROR(gckOS_CreateSignal(os, gcvTRUE, &signal)); |
| |
| /* Append the EVENT command to trigger the signal. */ |
| gcmkONERROR(gckEVENT_Signal(eventObject, signal, gcvKERNEL_PIXEL)); |
| |
| /* Submit the event queue. */ |
| gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower)); |
| |
| gcmkDUMP(Command->os, "#[kernel.stall]"); |
| |
| if (status == gcvSTATUS_CHIP_NOT_READY) |
| { |
| /* Error. */ |
| goto OnError; |
| } |
| |
| do |
| { |
| /* Wait for the signal. */ |
| status = gckOS_WaitSignal(os, signal, !FromPower, gcdGPU_ADVANCETIMER); |
| |
| if (status == gcvSTATUS_TIMEOUT) |
| { |
| #if gcmIS_DEBUG(gcdDEBUG_CODE) |
| gctUINT32 idle; |
| |
| /* Read idle register. */ |
| gcmkVERIFY_OK(gckHARDWARE_GetIdle( |
| hardware, gcvFALSE, &idle |
| )); |
| |
| gcmkTRACE( |
| gcvLEVEL_ERROR, |
| "%s(%d): idle=%08x", |
| __FUNCTION__, __LINE__, idle |
| ); |
| |
| gcmkVERIFY_OK(gckOS_MemoryBarrier(os, gcvNULL)); |
| #endif |
| |
| /* Advance timer. */ |
| timer += gcdGPU_ADVANCETIMER; |
| } |
| else if (status == gcvSTATUS_INTERRUPTED) |
| { |
| gcmkONERROR(gcvSTATUS_INTERRUPTED); |
| } |
| |
| } |
| while (gcmIS_ERROR(status)); |
| |
| /* Bail out on timeout. */ |
| if (gcmIS_ERROR(status)) |
| { |
| /* Broadcast the stuck GPU. */ |
| gcmkONERROR(gckOS_Broadcast( |
| os, hardware, gcvBROADCAST_GPU_STUCK |
| )); |
| } |
| |
| /* Delete the signal. */ |
| gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); |
| |
| /* Success. */ |
| gcmkFOOTER_NO(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| if (signal != gcvNULL) |
| { |
| /* Free the signal. */ |
| gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| #if (gcdENABLE_3D) |
| static gceSTATUS |
| _AttachWaitLinkFECommand( |
| IN gckCOMMAND Command, |
| OUT gckCONTEXT * Context, |
| OUT gctSIZE_T * MaxState, |
| OUT gctUINT32 * NumStates, |
| IN gctUINT32 ProcessID |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=%p", Command); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Acquire the context switching mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContext, gcvINFINITE |
| )); |
| acquired = gcvTRUE; |
| |
| /* Construct a gckCONTEXT object. */ |
| gcmkONERROR(gckCONTEXT_Construct( |
| Command->os, |
| Command->kernel->hardware, |
| ProcessID, |
| Context |
| )); |
| |
| /* Return the number of states in the context. */ |
| * MaxState = (* Context)->maxState; |
| * NumStates = (* Context)->numStates; |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| |
| /* Success. */ |
| gcmkFOOTER_ARG("*Context=0x%x", *Context); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Release mutex. */ |
| if (acquired) |
| { |
| /* Release the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Attach |
| ** |
| ** Attach user process. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** gctUINT32 ProcessID |
| ** Current process ID. |
| ** |
| ** OUTPUT: |
| ** |
| ** gckCONTEXT * Context |
| ** Pointer to a variable that will receive a pointer to a new |
| ** gckCONTEXT object. |
| ** |
| ** gctSIZE_T * StateCount |
| ** Pointer to a variable that will receive the number of states |
| ** in the context buffer. |
| */ |
| gceSTATUS |
| gckCOMMAND_Attach( |
| IN gckCOMMAND Command, |
| OUT gckCONTEXT * Context, |
| OUT gctSIZE_T * MaxState, |
| OUT gctUINT32 * NumStates, |
| IN gctUINT32 ProcessID |
| ) |
| { |
| gctUINT32 allocationSize; |
| gctPOINTER pointer; |
| gceSTATUS status; |
| |
| if (Command->feType == gcvHW_FE_WAIT_LINK) |
| { |
| status = _AttachWaitLinkFECommand(Command, |
| Context, |
| MaxState, |
| NumStates, |
| ProcessID); |
| } |
| else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) |
| { |
| /* |
| * For mcfe, we only allocate context which is used to |
| * store profile counters. |
| */ |
| allocationSize = gcmSIZEOF(struct _gckCONTEXT); |
| |
| /* Allocate the object. */ |
| gckOS_Allocate(Command->os, allocationSize, &pointer); |
| if (!pointer) |
| { |
| return gcvSTATUS_OUT_OF_MEMORY; |
| } |
| *Context = pointer; |
| /* Reset the entire object. */ |
| gckOS_ZeroMemory(*Context, allocationSize); |
| |
| /* Initialize the gckCONTEXT object. */ |
| (*Context)->object.type = gcvOBJ_CONTEXT; |
| (*Context)->os = Command->os; |
| (*Context)->hardware = Command->kernel->hardware; |
| *MaxState = 0; |
| *NumStates = 0; |
| |
| status = gcvSTATUS_OK; |
| } |
| else |
| { |
| /* Nothing to do. */ |
| *Context = gcvNULL; |
| *MaxState = 0; |
| *NumStates = 0; |
| |
| status = gcvSTATUS_OK; |
| } |
| |
| return status; |
| } |
| #endif |
| |
| static gceSTATUS |
| _DetachWaitLinkFECommand( |
| IN gckCOMMAND Command, |
| IN gckCONTEXT Context |
| ) |
| { |
| gceSTATUS status; |
| gctBOOL acquired = gcvFALSE; |
| |
| gcmkHEADER_ARG("Command=%p Context=%p", Command, Context); |
| |
| /* Verify the arguments. */ |
| gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); |
| |
| /* Acquire the context switching mutex. */ |
| gcmkONERROR(gckOS_AcquireMutex( |
| Command->os, Command->mutexContext, gcvINFINITE |
| )); |
| acquired = gcvTRUE; |
| |
| /* Construct a gckCONTEXT object. */ |
| gcmkONERROR(gckCONTEXT_Destroy(Context)); |
| |
| if (Command->currContext == Context) |
| { |
| /* Detach from gckCOMMAND object if the destoryed context is current context. */ |
| Command->currContext = gcvNULL; |
| } |
| |
| /* Release the context switching mutex. */ |
| gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return gcvSTATUS_OK; |
| |
| OnError: |
| /* Release mutex. */ |
| if (acquired) |
| { |
| /* Release the context switching mutex. */ |
| gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); |
| acquired = gcvFALSE; |
| } |
| |
| /* Return the status. */ |
| gcmkFOOTER(); |
| return status; |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_Detach |
| ** |
| ** Detach user process. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** gckCONTEXT Context |
| ** Pointer to a gckCONTEXT object to be destroyed. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_Detach( |
| IN gckCOMMAND Command, |
| IN gckCONTEXT Context |
| ) |
| { |
| if (Command->feType == gcvHW_FE_WAIT_LINK) |
| { |
| return _DetachWaitLinkFECommand(Command, Context); |
| } |
| else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) |
| { |
| gcmkOS_SAFE_FREE(Context->os, Context); |
| return gcvSTATUS_OK; |
| } |
| else |
| { |
| /* Nothing to do. */ |
| return gcvSTATUS_OK; |
| } |
| } |
| |
| static void |
| _DumpBuffer( |
| IN gctPOINTER Buffer, |
| IN gctUINT32 GpuAddress, |
| IN gctSIZE_T Size |
| ) |
| { |
| gctSIZE_T i, line, left; |
| gctUINT32_PTR data = Buffer; |
| |
| line = Size / 32; |
| left = Size % 32; |
| |
| for (i = 0; i < line; i++) |
| { |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]); |
| data += 8; |
| GpuAddress += 8 * 4; |
| } |
| |
| switch(left) |
| { |
| case 28: |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6]); |
| break; |
| case 24: |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5]); |
| break; |
| case 20: |
| gcmkPRINT("%08X : %08X %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3], data[4]); |
| break; |
| case 16: |
| gcmkPRINT("%08X : %08X %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2], data[3]); |
| break; |
| case 12: |
| gcmkPRINT("%08X : %08X %08X %08X", |
| GpuAddress, data[0], data[1], data[2]); |
| break; |
| case 8: |
| gcmkPRINT("%08X : %08X %08X", |
| GpuAddress, data[0], data[1]); |
| break; |
| case 4: |
| gcmkPRINT("%08X : %08X", |
| GpuAddress, data[0]); |
| break; |
| default: |
| break; |
| } |
| } |
| |
| /******************************************************************************* |
| ** |
| ** gckCOMMAND_DumpExecutingBuffer |
| ** |
| ** Dump the command buffer which GPU is executing. |
| ** |
| ** INPUT: |
| ** |
| ** gckCOMMAND Command |
| ** Pointer to a gckCOMMAND object. |
| ** |
| ** OUTPUT: |
| ** |
| ** Nothing. |
| */ |
| gceSTATUS |
| gckCOMMAND_DumpExecutingBuffer( |
| IN gckCOMMAND Command |
| ) |
| { |
| gceSTATUS status; |
| gckVIDMEM_NODE nodeObject = gcvNULL; |
| gctUINT32 gpuAddress; |
| gctPOINTER entry = gcvNULL; |
| gckOS os = Command->os; |
| gckKERNEL kernel = Command->kernel; |
| gctUINT32 i; |
| gckQUEUE queue = &kernel->hardware->linkQueue; |
| gctSIZE_T bytes; |
| gctUINT32 offset; |
| gctPOINTER entryDump; |
| gctUINT8 processName[24] = {0}; |
| |
| gcmkPRINT("**************************\n"); |
| gcmkPRINT("**** COMMAND BUF DUMP ****\n"); |
| gcmkPRINT("**************************\n"); |
| |
| gcmkPRINT(" Submitted commit stamp = %lld", Command->commitStamp - 1); |
| gcmkPRINT(" Executed commit stamp = %lld", *(gctUINT64_PTR)Command->fence->logical); |
| |
| gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); |
| gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); |
| |
| gcmkPRINT("DMA Address 0x%08X", gpuAddress); |
| |
| /* Find GPU address in video memory list. */ |
| status = gckVIDMEM_NODE_Find(kernel, gpuAddress, &nodeObject, &offset); |
| |
| if (gcmIS_SUCCESS(status) && nodeObject->type == gcvVIDMEM_TYPE_COMMAND) |
| { |
| gcmkONERROR(gckVIDMEM_NODE_LockCPU( |
| kernel, |
| nodeObject, |
| gcvFALSE, |
| gcvFALSE, |
| &entryDump |
| )); |
| |
| gcmkVERIFY_OK(gckVIDMEM_NODE_GetSize( |
| kernel, |
| nodeObject, |
| &bytes |
| )); |
| |
| gcmkPRINT("Command buffer around 0x%08X:", gpuAddress); |
| |
| /* Align to 4096. */ |
| offset &= 0xfffff000; |
| gpuAddress &= 0xfffff000; |
| |
| /* Dump max 4096 bytes. */ |
| bytes = (bytes - offset) > 4096 ? 4096 : (bytes - offset); |
| |
| /* Kernel address of page where stall point stay. */ |
| entryDump = (gctUINT8_PTR)entryDump + offset; |
| |
| _DumpBuffer(entryDump, gpuAddress, bytes); |
| |
| gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( |
| kernel, |
| nodeObject, |
| 0, |
| gcvFALSE, |
| gcvFALSE |
| )); |
| } |
| else |
| { |
| gcmkPRINT("Can not find command buffer around 0x%08X.\n", gpuAddress); |
| } |
| |
| /* new line. */ |
| gcmkPRINT(""); |
| |
| gcmkPRINT("Kernel command buffers:"); |
| |
| for (i = 0; i < gcdCOMMAND_QUEUES; i++) |
| { |
| entry = Command->queues[i].logical; |
| gpuAddress = Command->queues[i].address; |
| |
| gcmkPRINT("command buffer %d at 0x%08X size %u", |
| i, gpuAddress, Command->pageSize); |
| _DumpBuffer(entry, gpuAddress, Command->pageSize); |
| } |
| |
| /* new line. */ |
| gcmkPRINT(""); |
| |
| if (queue->count) |
| { |
| gcmkPRINT("Dump Level is %d, dump %d valid record in link queue:", |
| Command->kernel->stuckDump, queue->count); |
| } |
| |
| for (i = 0; i < queue->count; i++) |
| { |
| gcuQUEUEDATA * queueData; |
| gckLINKDATA linkData; |
| |
| gckQUEUE_GetData(queue, i, &queueData); |
| |
| linkData = &queueData->linkData; |
| |
| /* Get gpu address of this command buffer. */ |
| gpuAddress = linkData->start; |
| bytes = linkData->end - gpuAddress; |
| |
| processName[0] = '\0'; |
| gckOS_GetProcessNameByPid(linkData->pid, sizeof(processName), processName); |
| |
| gcmkPRINT("Link record %d: [%08X - %08X] from command %08X %08X pid %u (%s):", |
| i, |
| linkData->start, |
| linkData->end, |
| linkData->linkLow, |
| linkData->linkHigh, |
| linkData->pid, |
| processName); |
| |
| /* Find GPU address in video memory list. */ |
| status = gckVIDMEM_NODE_Find(kernel, gpuAddress, &nodeObject, &offset); |
| |
| if (gcmIS_SUCCESS(status) && nodeObject->type == gcvVIDMEM_TYPE_COMMAND) |
| { |
| gcmkONERROR(gckVIDMEM_NODE_LockCPU( |
| kernel, |
| nodeObject, |
| gcvFALSE, |
| gcvFALSE, |
| &entryDump |
| )); |
| |
| /* Kernel address of page where stall point stay. */ |
| entryDump = (gctUINT8_PTR)entryDump + offset; |
| |
| _DumpBuffer(entryDump, gpuAddress, bytes); |
| |
| gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( |
| kernel, |
| nodeObject, |
| 0, |
| gcvFALSE, |
| gcvFALSE |
| )); |
| } |
| else |
| { |
| gcmkPRINT("Not found"); |
| } |
| |
| /* new line. */ |
| gcmkPRINT(""); |
| } |
| |
| return gcvSTATUS_OK; |
| |
| OnError: |
| return status; |
| } |
| |