Google Git Google Git
Sign in
coral/linux-imx/refs/heads/master/./drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel.c
blob: 5260aef60149f57b9e9e970444351ab5f09c2570 [file] [edit]
/****************************************************************************
*
* The MIT License (MIT)
*
* Copyright (c) 2014 - 2018 Vivante Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
*****************************************************************************
*
* The GPL License (GPL)
*
* Copyright (C) 2014 - 2018 Vivante Corporation
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*****************************************************************************
*
* Note: This software is released under dual MIT and GPL licenses. A
* recipient may use this file under the terms of either the MIT license or
* GPL License. If you wish to use only one license not the other, you can
* indicate your decision by deleting one of the above license notices in your
* version of this file.
*
*****************************************************************************/
#include "gc_hal_kernel_precomp.h"
#if gcdDEC_ENABLE_AHB
#include "viv_dec300_main.h"
#endif
#define _GC_OBJ_ZONE gcvZONE_KERNEL
/*******************************************************************************
***** Version Signature *******************************************************/
#define _gcmTXT2STR(t) #t
#define gcmTXT2STR(t) _gcmTXT2STR(t)
const char * _VERSION = "\n\0$VERSION$"
gcmTXT2STR(gcvVERSION_MAJOR) "."
gcmTXT2STR(gcvVERSION_MINOR) "."
gcmTXT2STR(gcvVERSION_PATCH) ":"
gcmTXT2STR(gcvVERSION_BUILD) "$\n";
/******************************************************************************\
******************************* gckKERNEL API Code ******************************
\******************************************************************************/
#if gcmIS_DEBUG(gcdDEBUG_TRACE)
#define gcmDEFINE2TEXT(d) #d
gctCONST_STRING _DispatchText[] =
{
gcmDEFINE2TEXT(gcvHAL_QUERY_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_QUERY_CHIP_IDENTITY),
gcmDEFINE2TEXT(gcvHAL_QUERY_CHIP_FREQUENCY),
gcmDEFINE2TEXT(gcvHAL_ALLOCATE_NON_PAGED_MEMORY),
gcmDEFINE2TEXT(gcvHAL_FREE_NON_PAGED_MEMORY),
gcmDEFINE2TEXT(gcvHAL_ALLOCATE_CONTIGUOUS_MEMORY),
gcmDEFINE2TEXT(gcvHAL_FREE_CONTIGUOUS_MEMORY),
gcmDEFINE2TEXT(gcvHAL_ALLOCATE_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_ALLOCATE_LINEAR_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_RELEASE_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_MAP_MEMORY),
gcmDEFINE2TEXT(gcvHAL_UNMAP_MEMORY),
gcmDEFINE2TEXT(gcvHAL_MAP_USER_MEMORY),
gcmDEFINE2TEXT(gcvHAL_UNMAP_USER_MEMORY),
gcmDEFINE2TEXT(gcvHAL_LOCK_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_UNLOCK_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_EVENT_COMMIT),
gcmDEFINE2TEXT(gcvHAL_USER_SIGNAL),
gcmDEFINE2TEXT(gcvHAL_SIGNAL),
gcmDEFINE2TEXT(gcvHAL_WRITE_DATA),
gcmDEFINE2TEXT(gcvHAL_COMMIT),
gcmDEFINE2TEXT(gcvHAL_STALL),
gcmDEFINE2TEXT(gcvHAL_READ_REGISTER),
gcmDEFINE2TEXT(gcvHAL_WRITE_REGISTER),
gcmDEFINE2TEXT(gcvHAL_GET_PROFILE_SETTING),
gcmDEFINE2TEXT(gcvHAL_SET_PROFILE_SETTING),
gcmDEFINE2TEXT(gcvHAL_PROFILE_REGISTERS_2D),
gcmDEFINE2TEXT(gcvHAL_READ_ALL_PROFILE_REGISTERS_PART1),
gcmDEFINE2TEXT(gcvHAL_READ_ALL_PROFILE_REGISTERS_PART2),
gcmDEFINE2TEXT(gcvHAL_READ_PROFILER_REGISTER_SETTING),
gcmDEFINE2TEXT(gcvHAL_SET_POWER_MANAGEMENT_STATE),
gcmDEFINE2TEXT(gcvHAL_QUERY_POWER_MANAGEMENT_STATE),
gcmDEFINE2TEXT(gcvHAL_GET_BASE_ADDRESS),
gcmDEFINE2TEXT(gcvHAL_SET_IDLE),
gcmDEFINE2TEXT(gcvHAL_QUERY_KERNEL_SETTINGS),
gcmDEFINE2TEXT(gcvHAL_RESET),
gcmDEFINE2TEXT(gcvHAL_MAP_PHYSICAL),
gcmDEFINE2TEXT(gcvHAL_DEBUG),
gcmDEFINE2TEXT(gcvHAL_CACHE),
gcmDEFINE2TEXT(gcvHAL_TIMESTAMP),
gcmDEFINE2TEXT(gcvHAL_DATABASE),
gcmDEFINE2TEXT(gcvHAL_VERSION),
gcmDEFINE2TEXT(gcvHAL_CHIP_INFO),
gcmDEFINE2TEXT(gcvHAL_ATTACH),
gcmDEFINE2TEXT(gcvHAL_DETACH),
gcmDEFINE2TEXT(gcvHAL_SET_TIMEOUT),
gcmDEFINE2TEXT(gcvHAL_GET_FRAME_INFO),
gcmDEFINE2TEXT(gcvHAL_DUMP_GPU_PROFILE),
gcmDEFINE2TEXT(gcvHAL_QUERY_COMMAND_BUFFER),
gcmDEFINE2TEXT(gcvHAL_COMMIT_DONE),
gcmDEFINE2TEXT(gcvHAL_DUMP_GPU_STATE),
gcmDEFINE2TEXT(gcvHAL_DUMP_EVENT),
gcmDEFINE2TEXT(gcvHAL_ALLOCATE_VIRTUAL_COMMAND_BUFFER),
gcmDEFINE2TEXT(gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER),
gcmDEFINE2TEXT(gcvHAL_SET_FSCALE_VALUE),
gcmDEFINE2TEXT(gcvHAL_GET_FSCALE_VALUE),
gcmDEFINE2TEXT(gcvHAL_EXPORT_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_NAME_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_IMPORT_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_QUERY_RESET_TIME_STAMP),
gcmDEFINE2TEXT(gcvHAL_READ_REGISTER_EX),
gcmDEFINE2TEXT(gcvHAL_WRITE_REGISTER_EX),
gcmDEFINE2TEXT(gcvHAL_CREATE_NATIVE_FENCE),
gcmDEFINE2TEXT(gcvHAL_WAIT_NATIVE_FENCE),
gcmDEFINE2TEXT(gcvHAL_DESTROY_MMU),
gcmDEFINE2TEXT(gcvHAL_SHBUF),
gcmDEFINE2TEXT(gcvHAL_GET_GRAPHIC_BUFFER_FD),
gcmDEFINE2TEXT(gcvHAL_SET_VIDEO_MEMORY_METADATA),
gcmDEFINE2TEXT(gcvHAL_GET_VIDEO_MEMORY_FD),
gcmDEFINE2TEXT(gcvHAL_CONFIG_POWER_MANAGEMENT),
gcmDEFINE2TEXT(gcvHAL_WRAP_USER_MEMORY),
gcmDEFINE2TEXT(gcvHAL_WAIT_FENCE),
#if gcdDEC_ENABLE_AHB
gcmDEFINE2TEXT(gcvHAL_DEC300_READ),
gcmDEFINE2TEXT(gcvHAL_DEC300_WRITE),
gcmDEFINE2TEXT(gcvHAL_DEC300_FLUSH),
gcmDEFINE2TEXT(gcvHAL_DEC300_FLUSH_WAIT),
#endif
gcmDEFINE2TEXT(gcvHAL_BOTTOM_HALF_UNLOCK_VIDEO_MEMORY),
gcmDEFINE2TEXT(gcvHAL_QUERY_CHIP_OPTION),
};
#endif
#if gcdGPU_TIMEOUT && gcdINTERRUPT_STATISTIC
void
_MonitorTimerFunction(
gctPOINTER Data
)
{
gckKERNEL kernel = (gckKERNEL)Data;
gctINT32 pendingInterrupt;
gctBOOL reset = gcvFALSE;
gctINT32 mask;
gctUINT32 advance = kernel->timeOut/2;
#if gcdENABLE_VG
if (kernel->core == gcvCORE_VG)
{
return;
}
#endif
if (kernel->monitorTimerStop)
{
/* Stop. */
return;
}
gckOS_AtomGet(kernel->os, kernel->eventObj->interruptCount, &pendingInterrupt);
if (pendingInterrupt < 0)
{
gctINT i = 0 - pendingInterrupt;
gctINT pendingMask;
gcmkVERIFY_OK(gckOS_AtomGet(
kernel->os,
kernel->hardware->pendingEvent,
&pendingMask
));
gcmkPRINT("[galcore]: Number of pending interrupt is %d mask is %x",
pendingInterrupt, pendingMask);
while (i--)
{
/* Ignore counting which should not exist. */
gckOS_AtomIncrement(kernel->os, kernel->eventObj->interruptCount, &pendingInterrupt);
}
gckOS_AtomGet(kernel->os, kernel->eventObj->interruptCount, &pendingInterrupt);
}
if (kernel->monitoring == gcvFALSE)
{
if (pendingInterrupt)
{
/* Begin to mointor GPU state. */
kernel->monitoring = gcvTRUE;
/* Record current state. */
kernel->lastCommitStamp = kernel->eventObj->lastCommitStamp;
kernel->restoreAddress = kernel->hardware->lastWaitLink;
gcmkVERIFY_OK(gckOS_AtomGet(
kernel->os,
kernel->hardware->pendingEvent,
&kernel->restoreMask
));
/* Clear timeout. */
kernel->timer = 0;
}
}
else
{
if (pendingInterrupt)
{
gcmkVERIFY_OK(gckOS_AtomGet(
kernel->os,
kernel->hardware->pendingEvent,
&mask
));
if (kernel->eventObj->lastCommitStamp == kernel->lastCommitStamp
&& kernel->hardware->lastWaitLink == kernel->restoreAddress
&& mask == kernel->restoreMask
)
{
/* GPU state is not changed, accumlate timeout. */
kernel->timer += advance;
if (kernel->timer >= kernel->timeOut)
{
/* GPU stuck, trigger reset. */
reset = gcvTRUE;
}
}
else
{
/* GPU state changed, cancel current timeout.*/
kernel->monitoring = gcvFALSE;
}
}
else
{
/* GPU finish all jobs, cancel current timeout*/
kernel->monitoring = gcvFALSE;
}
}
if (reset)
{
gckKERNEL_Recovery(kernel);
/* Work in this timeout is done. */
kernel->monitoring = gcvFALSE;
}
gcmkVERIFY_OK(gckOS_StartTimer(kernel->os, kernel->monitorTimer, advance));
}
#endif
#if gcdPROCESS_ADDRESS_SPACE
gceSTATUS
_MapCommandBuffer(
IN gckKERNEL Kernel
)
{
gceSTATUS status;
gctUINT32 i;
gctPHYS_ADDR_T physical;
gctUINT32 address;
gckMMU mmu;
gcmkONERROR(gckKERNEL_GetProcessMMU(Kernel, &mmu));
for (i = 0; i < gcdCOMMAND_QUEUES; i++)
{
gcmkONERROR(gckOS_GetPhysicalAddress(
Kernel->os,
Kernel->command->queues[i].logical,
&physical
));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Kernel->os, physical, &physical));
gcmkSAFECASTPHYSADDRT(address, physical);
gcmkONERROR(gckMMU_FlatMapping(mmu, address, 1));
}
return gcvSTATUS_OK;
OnError:
return status;
}
#endif
void
_DumpDriverConfigure(
IN gckKERNEL Kernel
)
{
gcmkPRINT_N(0, "**************************\n");
gcmkPRINT_N(0, "*** GPU DRV CONFIG ***\n");
gcmkPRINT_N(0, "**************************\n");
gcmkPRINT("Galcore version %d.%d.%d.%d\n",
gcvVERSION_MAJOR, gcvVERSION_MINOR, gcvVERSION_PATCH, gcvVERSION_BUILD);
gckOS_DumpParam();
}
void
_DumpState(
IN gckKERNEL Kernel
)
{
/* Dump GPU Debug registers. */
gcmkVERIFY_OK(gckHARDWARE_DumpGPUState(Kernel->hardware));
gcmkVERIFY_OK(gckCOMMAND_DumpExecutingBuffer(Kernel->command));
/* Dump Pending event. */
gcmkVERIFY_OK(gckEVENT_Dump(Kernel->eventObj));
/* Dump Process DB. */
gcmkVERIFY_OK(gckKERNEL_DumpProcessDB(Kernel));
#if gcdRECORD_COMMAND
/* Dump record. */
gckRECORDER_Dump(Kernel->command->recorder);
#endif
}
static gceHARDWARE_TYPE
_GetHardwareType(
IN gckKERNEL Kernel
)
{
gceHARDWARE_TYPE type;
gcmkHEADER();
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
#if gcdENABLE_VG
if (Kernel->vg)
{
type = gcvHARDWARE_VG;
}
else
#endif
{
type = Kernel->hardware->type;
}
gcmkFOOTER_ARG("type=%d", type);
return type;
}
gceSTATUS
_SetRecovery(
IN gckKERNEL Kernel,
IN gctBOOL Recovery,
IN gctUINT32 StuckDump
)
{
Kernel->recovery = Recovery;
if (Recovery == gcvFALSE)
{
/* Dump stuck information if Recovery is disabled. */
Kernel->stuckDump = gcmMAX(StuckDump, gcvSTUCK_DUMP_USER_COMMAND);
}
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckKERNEL_Construct
**
** Construct a new gckKERNEL object.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gceCORE Core
** Specified core.
**
** IN gctPOINTER Context
** Pointer to a driver defined context.
**
** IN gckDB SharedDB,
** Pointer to a shared DB.
**
** OUTPUT:
**
** gckKERNEL * Kernel
** Pointer to a variable that will hold the pointer to the gckKERNEL
** object.
*/
gceSTATUS
gckKERNEL_Construct(
IN gckOS Os,
IN gceCORE Core,
IN gctUINT ChipID,
IN gctPOINTER Context,
IN gckDEVICE Device,
IN gckDB SharedDB,
OUT gckKERNEL * Kernel
)
{
gckKERNEL kernel = gcvNULL;
gceSTATUS status;
gctSIZE_T i;
gctPOINTER pointer = gcvNULL;
gctUINT32 recovery;
gctUINT32 stuckDump;
gcmkHEADER_ARG("Os=0x%x Context=0x%x", Os, Context);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Kernel != gcvNULL);
/* Allocate the gckKERNEL object. */
gcmkONERROR(gckOS_Allocate(Os,
gcmSIZEOF(struct _gckKERNEL),
&pointer));
/* Zero the object. */
gckOS_ZeroMemory(pointer, gcmSIZEOF(struct _gckKERNEL));
kernel = pointer;
/* Initialize the gckKERNEL object. */
kernel->object.type = gcvOBJ_KERNEL;
kernel->os = Os;
kernel->core = Core;
kernel->device = Device;
kernel->chipID = ChipID;
#if gcdENABLE_TRUST_APPLICATION
/* Connect to security service for this GPU. */
gcmkONERROR(gckKERNEL_SecurityOpen(kernel, kernel->core, &kernel->securityChannel));
#endif
if (SharedDB == gcvNULL)
{
gcmkONERROR(gckOS_Allocate(Os,
gcmSIZEOF(struct _gckDB),
&pointer));
kernel->db = pointer;
kernel->dbCreated = gcvTRUE;
kernel->db->freeDatabase = gcvNULL;
kernel->db->freeRecord = gcvNULL;
kernel->db->dbMutex = gcvNULL;
kernel->db->lastDatabase = gcvNULL;
kernel->db->idleTime = 0;
kernel->db->lastIdle = 0;
kernel->db->lastSlowdown = 0;
for (i = 0; i < gcmCOUNTOF(kernel->db->db); ++i)
{
kernel->db->db[i] = gcvNULL;
}
/* Construct a database mutex. */
gcmkONERROR(gckOS_CreateMutex(Os, &kernel->db->dbMutex));
/* Construct a video memory name database. */
gcmkONERROR(gckKERNEL_CreateIntegerDatabase(kernel, &kernel->db->nameDatabase));
/* Construct a video memory name database mutex. */
gcmkONERROR(gckOS_CreateMutex(Os, &kernel->db->nameDatabaseMutex));
/* Construct a pointer name database. */
gcmkONERROR(gckKERNEL_CreateIntegerDatabase(kernel, &kernel->db->pointerDatabase));
/* Construct a pointer name database mutex. */
gcmkONERROR(gckOS_CreateMutex(Os, &kernel->db->pointerDatabaseMutex));
/* Initialize on fault vidmem list. */
gcsLIST_Init(&kernel->db->onFaultVidmemList);
}
else
{
kernel->db = SharedDB;
kernel->dbCreated = gcvFALSE;
}
for (i = 0; i < gcmCOUNTOF(kernel->timers); ++i)
{
kernel->timers[i].startTime = 0;
kernel->timers[i].stopTime = 0;
}
/* Save context. */
kernel->context = Context;
/* Construct atom holding number of clients. */
kernel->atomClients = gcvNULL;
gcmkONERROR(gckOS_AtomConstruct(Os, &kernel->atomClients));
kernel->recovery = gcvTRUE;
kernel->stuckDump = gcvSTUCK_DUMP_NONE;
/* Override default recovery and stuckDump setting. */
status = gckOS_QueryOption(Os, "recovery", &recovery);
if (gcmIS_SUCCESS(status))
{
status = gckOS_QueryOption(Os, "stuckDump", &stuckDump);
gcmkASSERT(status == gcvSTATUS_OK);
_SetRecovery(kernel, recovery, stuckDump);
}
/* Need the kernel reference before gckKERNEL_Construct() completes.
gckOS_MapPagesEx() is called to map kernel virtual command buffers. */
*Kernel = kernel;
kernel->virtualBufferHead =
kernel->virtualBufferTail = gcvNULL;
gcmkONERROR(
gckOS_CreateMutex(Os, (gctPOINTER)&kernel->virtualBufferLock));
#if gcdENABLE_VG
kernel->vg = gcvNULL;
if (Core == gcvCORE_VG)
{
gctUINT32 contiguousBase;
gctUINT32 contiguousSize = 0;
/* Construct the gckMMU object. */
gcmkONERROR(
gckVGKERNEL_Construct(Os, Context, kernel, &kernel->vg));
kernel->timeOut = gcdGPU_TIMEOUT;
status = gckOS_QueryOption(Os, "contiguousBase", &contiguousBase);
if (gcmIS_SUCCESS(status))
{
status = gckOS_QueryOption(Os, "contiguousSize", &contiguousSize);
}
if (gcmIS_SUCCESS(status) && contiguousSize)
{
gctUINT64 gpuContiguousBase;
gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(Os, contiguousBase, &gpuContiguousBase));
gcmkSAFECASTPHYSADDRT(kernel->contiguousBaseAddress, gpuContiguousBase);
}
}
else
#endif
{
/* Construct the gckHARDWARE object. */
gcmkONERROR(
gckHARDWARE_Construct(Os, kernel->core, &kernel->hardware));
/* Set pointer to gckKERNEL object in gckHARDWARE object. */
kernel->hardware->kernel = kernel;
kernel->timeOut = kernel->hardware->type == gcvHARDWARE_2D
? gcdGPU_2D_TIMEOUT
: gcdGPU_TIMEOUT
;
/* Initialize virtual command buffer. */
#if gcdALLOC_CMD_FROM_RESERVE || gcdSECURITY || gcdDISABLE_GPU_VIRTUAL_ADDRESS || !USE_KERNEL_VIRTUAL_BUFFERS
kernel->virtualCommandBuffer = gcvFALSE;
#else
kernel->virtualCommandBuffer = kernel->hardware->options.enableMMU;
#endif
#if gcdSHARED_PAGETABLE
/* Construct the gckMMU object. */
gcmkONERROR(
gckMMU_Construct(kernel, gcdMMU_SIZE, &kernel->mmu));
#else
if (Device == gcvNULL)
{
/* Construct the gckMMU object. */
gcmkONERROR(
gckMMU_Construct(kernel, gcdMMU_SIZE, &kernel->mmu));
}
else
{
gcmkONERROR(gckDEVICE_GetMMU(Device, kernel->hardware->type, &kernel->mmu));
if (kernel->mmu == gcvNULL)
{
gcmkONERROR(
gckMMU_Construct(kernel, gcdMMU_SIZE, &kernel->mmu));
gcmkONERROR(
gckDEVICE_SetMMU(Device, kernel->hardware->type, kernel->mmu));
}
}
gcmkVERIFY_OK(gckMMU_AttachHardware(kernel->mmu, kernel->hardware));
#endif
kernel->contiguousBaseAddress = kernel->mmu->contiguousBaseAddress;
kernel->externalBaseAddress = kernel->mmu->externalBaseAddress;
/* Construct the gckCOMMAND object. */
gcmkONERROR(
gckCOMMAND_Construct(kernel, &kernel->command));
if (gckHARDWARE_IsFeatureAvailable(kernel->hardware, gcvFEATURE_ASYNC_BLIT))
{
/* Construct the gckASYNC_COMMAND object for BLT engine. */
gcmkONERROR(gckASYNC_COMMAND_Construct(kernel, &kernel->asyncCommand));
/* Construct gckEVENT for BLT. */
gcmkONERROR(gckEVENT_Construct(kernel, &kernel->asyncEvent));
kernel->asyncEvent->asyncCommand = kernel->asyncCommand;
kernel->command->asyncCommand = kernel->asyncCommand;
}
/* Construct the gckEVENT object. */
gcmkONERROR(
gckEVENT_Construct(kernel, &kernel->eventObj));
gcmkVERIFY_OK(gckOS_GetTime(&kernel->resetTimeStamp));
gcmkONERROR(gckHARDWARE_PrepareFunctions(kernel->hardware));
/* Initialize the hardware. */
gcmkONERROR(
gckHARDWARE_InitializeHardware(kernel->hardware));
#if gcdDVFS
if (gckHARDWARE_IsFeatureAvailable(kernel->hardware,
gcvFEATURE_DYNAMIC_FREQUENCY_SCALING))
{
gcmkONERROR(gckDVFS_Construct(kernel->hardware, &kernel->dvfs));
gcmkONERROR(gckDVFS_Start(kernel->dvfs));
}
#endif
#if COMMAND_PROCESSOR_VERSION == 1
/* Start the command queue. */
gcmkONERROR(gckCOMMAND_Start(kernel->command));
#endif
}
#if VIVANTE_PROFILER
/* Initialize profile setting */
kernel->profileEnable = gcvFALSE;
kernel->profileCleanRegister = gcvTRUE;
#endif
#if gcdLINUX_SYNC_FILE
gcmkONERROR(gckOS_CreateSyncTimeline(Os, Core, &kernel->timeline));
#endif
#if gcdSECURITY
/* Connect to security service for this GPU. */
gcmkONERROR(gckKERNEL_SecurityOpen(kernel, kernel->core, &kernel->securityChannel));
#endif
#if gcdGPU_TIMEOUT && gcdINTERRUPT_STATISTIC
if (kernel->timeOut)
{
gcmkVERIFY_OK(gckOS_CreateTimer(
Os,
(gctTIMERFUNCTION)_MonitorTimerFunction,
(gctPOINTER)kernel,
&kernel->monitorTimer
));
kernel->monitoring = gcvFALSE;
kernel->monitorTimerStop = gcvFALSE;
gcmkVERIFY_OK(gckOS_StartTimer(
Os,
kernel->monitorTimer,
100
));
}
#endif
/* Return pointer to the gckKERNEL object. */
*Kernel = kernel;
/* Success. */
gcmkFOOTER_ARG("*Kernel=0x%x", *Kernel);
return gcvSTATUS_OK;
OnError:
*Kernel = gcvNULL;
if (kernel != gcvNULL)
{
gckKERNEL_Destroy(kernel);
}
/* Return the error. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_Destroy
**
** Destroy an gckKERNEL object.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object to destroy.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckKERNEL_Destroy(
IN gckKERNEL Kernel
)
{
gctSIZE_T i;
gcsDATABASE_PTR database, databaseNext;
gcsDATABASE_RECORD_PTR record, recordNext;
gcmkHEADER_ARG("Kernel=0x%x", Kernel);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
#if QNX_SINGLE_THREADED_DEBUGGING
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->debugMutex));
#endif
/* Destroy the database. */
if (Kernel->dbCreated)
{
for (i = 0; i < gcmCOUNTOF(Kernel->db->db); ++i)
{
if (Kernel->db->db[i] != gcvNULL)
{
gcmkVERIFY_OK(
gckKERNEL_DestroyProcessDB(Kernel, Kernel->db->db[i]->processID));
}
}
/* Free all databases. */
for (database = Kernel->db->freeDatabase;
database != gcvNULL;
database = databaseNext)
{
databaseNext = database->next;
if (database->counterMutex)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, database->counterMutex));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, database));
}
if (Kernel->db->lastDatabase != gcvNULL)
{
if (Kernel->db->lastDatabase->counterMutex)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->lastDatabase->counterMutex));
}
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Kernel->db->lastDatabase));
}
/* Free all database records. */
for (record = Kernel->db->freeRecord; record != gcvNULL; record = recordNext)
{
recordNext = record->next;
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, record));
}
if (Kernel->db->dbMutex)
{
/* Destroy the database mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->dbMutex));
}
if (Kernel->db->nameDatabase)
{
/* Destroy video memory name database. */
gcmkVERIFY_OK(gckKERNEL_DestroyIntegerDatabase(Kernel, Kernel->db->nameDatabase));
}
if (Kernel->db->nameDatabaseMutex)
{
/* Destroy video memory name database mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->nameDatabaseMutex));
}
if (Kernel->db->pointerDatabase)
{
/* Destroy id-pointer database. */
gcmkVERIFY_OK(gckKERNEL_DestroyIntegerDatabase(Kernel, Kernel->db->pointerDatabase));
}
if (Kernel->db->pointerDatabaseMutex)
{
/* Destroy id-pointer database mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
}
if (Kernel->db)
{
/* Destroy the database. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Kernel->db));
}
/* Notify stuck timer to quit. */
Kernel->monitorTimerStop = gcvTRUE;
}
if (Kernel->monitorTimer)
{
/* Stop and destroy monitor timer. */
gcmkVERIFY_OK(gckOS_StopTimer(Kernel->os, Kernel->monitorTimer));
gcmkVERIFY_OK(gckOS_DestroyTimer(Kernel->os, Kernel->monitorTimer));
}
#if gcdENABLE_VG
if (Kernel->vg)
{
gcmkVERIFY_OK(gckVGKERNEL_Destroy(Kernel->vg));
}
else
#endif
{
if (Kernel->command)
{
/* Destroy the gckCOMMNAND object. */
gcmkVERIFY_OK(gckCOMMAND_Destroy(Kernel->command));
}
if (Kernel->asyncCommand)
{
gcmkVERIFY_OK(gckASYNC_COMMAND_Destroy(Kernel->asyncCommand));
}
if (Kernel->asyncEvent)
{
gcmkVERIFY_OK(gckEVENT_Destroy(Kernel->asyncEvent));
}
if (Kernel->eventObj)
{
/* Destroy the gckEVENT object. */
gcmkVERIFY_OK(gckEVENT_Destroy(Kernel->eventObj));
}
gcmkVERIFY_OK(gckHARDWARE_DestroyFunctions(Kernel->hardware));
if (Kernel->mmu)
{
#if gcdSHARED_PAGETABLE
/* Destroy the gckMMU object. */
gcmkVERIFY_OK(gckMMU_Destroy(Kernel->mmu));
#else
if (Kernel->mmu->hardware == Kernel->hardware)
{
/* Destroy the gckMMU object. */
gcmkVERIFY_OK(gckMMU_Destroy(Kernel->mmu));
}
#endif
}
if (Kernel->hardware)
{
/* Destroy the gckHARDWARE object. */
gcmkVERIFY_OK(gckHARDWARE_Destroy(Kernel->hardware));
}
}
if (Kernel->atomClients)
{
/* Detsroy the client atom. */
gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, Kernel->atomClients));
}
if (Kernel->virtualBufferLock)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->virtualBufferLock));
}
#if gcdDVFS
if (Kernel->dvfs)
{
gcmkVERIFY_OK(gckDVFS_Stop(Kernel->dvfs));
gcmkVERIFY_OK(gckDVFS_Destroy(Kernel->dvfs));
}
#endif
#if gcdLINUX_SYNC_FILE
if (Kernel->timeline)
{
gcmkVERIFY_OK(gckOS_DestroySyncTimeline(Kernel->os, Kernel->timeline));
}
#endif
#if gcdSECURITY
if (Kernel->securityChannel)
{
gcmkVERIFY_OK(gckKERNEL_SecurityClose(Kernel->securityChannel));
}
#endif
/* Mark the gckKERNEL object as unknown. */
Kernel->object.type = gcvOBJ_UNKNOWN;
/* Free the gckKERNEL object. */
gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Kernel));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** _AllocateMemory
**
** Private function to walk all required memory pools to allocate the requested
** amount of video memory.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gcsHAL_INTERFACE * Interface
** Pointer to a gcsHAL_INTERFACE structure that defines the command to
** be dispatched.
**
** OUTPUT:
**
** gcsHAL_INTERFACE * Interface
** Pointer to a gcsHAL_INTERFACE structure that receives any data to be
** returned.
*/
gceSTATUS
gckKERNEL_AllocateLinearMemory(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN OUT gcePOOL * Pool,
IN gctSIZE_T Bytes,
IN gctUINT32 Alignment,
IN gceSURF_TYPE Type,
IN gctUINT32 Flag,
OUT gctUINT32 * Node
)
{
gcePOOL pool;
gceSTATUS status;
gckVIDMEM videoMemory;
gctINT loopCount;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gctBOOL tileStatusInVirtual;
gctBOOL contiguous = gcvFALSE;
gctBOOL cacheable = gcvFALSE;
gctBOOL secure = gcvFALSE;
gctBOOL fastPools = gcvFALSE;
gctBOOL hasFastPools = gcvFALSE;
gctSIZE_T bytes = Bytes;
gctUINT32 handle = 0;
gceDATABASE_TYPE type;
gcmkHEADER_ARG("Kernel=0x%x *Pool=%d Bytes=%lu Alignment=%lu Type=%d",
Kernel, *Pool, Bytes, Alignment, Type);
gcmkVERIFY_ARGUMENT(Pool != gcvNULL);
gcmkVERIFY_ARGUMENT(Bytes != 0);
/* Get basic type. */
Type &= 0xFF;
/* Check flags. */
contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS;
cacheable = Flag & gcvALLOC_FLAG_CACHEABLE;
secure = Flag & gcvALLOC_FLAG_SECURITY;
if (Flag & gcvALLOC_FLAG_FAST_POOLS)
{
fastPools = gcvTRUE;
Flag &= ~gcvALLOC_FLAG_FAST_POOLS;
}
#if gcdALLOC_ON_FAULT
if (Type == gcvSURF_RENDER_TARGET)
{
Flag |= gcvALLOC_FLAG_ALLOC_ON_FAULT;
}
#endif
if (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT)
{
*Pool = gcvPOOL_VIRTUAL;
}
if (Flag & gcvALLOC_FLAG_DMABUF_EXPORTABLE)
{
gctSIZE_T pageSize = 0;
gckOS_GetPageSize(Kernel->os, &pageSize);
/* Usually, the exported dmabuf might be later imported to DRM,
** while DRM requires input size to be page aligned.
*/
Bytes = gcmALIGN(Bytes, pageSize);
}
AllocateMemory:
/* Get initial pool. */
switch (pool = *Pool)
{
case gcvPOOL_DEFAULT:
case gcvPOOL_LOCAL:
pool = gcvPOOL_LOCAL_INTERNAL;
loopCount = (gctINT) gcvPOOL_NUMBER_OF_POOLS;
break;
case gcvPOOL_UNIFIED:
pool = gcvPOOL_SYSTEM;
loopCount = (gctINT) gcvPOOL_NUMBER_OF_POOLS;
break;
case gcvPOOL_CONTIGUOUS:
loopCount = (gctINT) gcvPOOL_NUMBER_OF_POOLS;
break;
default:
loopCount = 1;
break;
}
while (loopCount-- > 0)
{
if (pool == gcvPOOL_VIRTUAL)
{
/* Create a gcuVIDMEM_NODE for virtual memory. */
gcmkONERROR(
gckVIDMEM_ConstructVirtual(Kernel, Flag | gcvALLOC_FLAG_NON_CONTIGUOUS, Bytes, &node));
bytes = node->Virtual.bytes;
node->Virtual.type = Type;
/* Success. */
break;
}
else
if (pool == gcvPOOL_CONTIGUOUS)
{
#if gcdCONTIGUOUS_SIZE_LIMIT
if (Bytes > gcdCONTIGUOUS_SIZE_LIMIT && contiguous == gcvFALSE)
{
status = gcvSTATUS_OUT_OF_MEMORY;
}
else
#endif
{
/* Create a gcuVIDMEM_NODE from contiguous memory. */
status = gckVIDMEM_ConstructVirtual(
Kernel,
Flag | gcvALLOC_FLAG_CONTIGUOUS,
Bytes,
&node);
}
if (gcmIS_SUCCESS(status))
{
bytes = node->Virtual.bytes;
node->Virtual.type = Type;
/* Memory allocated. */
break;
}
}
else
/* gcvPOOL_SYSTEM can't be cacheable. */
if (cacheable == gcvFALSE && secure == gcvFALSE)
{
/* Get pointer to gckVIDMEM object for pool. */
status = gckKERNEL_GetVideoMemoryPool(Kernel, pool, &videoMemory);
if (gcmIS_SUCCESS(status))
{
/* Allocate memory. */
#if defined(gcdLINEAR_SIZE_LIMIT)
/* 512 KB */
if (Bytes > gcdLINEAR_SIZE_LIMIT)
{
status = gcvSTATUS_OUT_OF_MEMORY;
}
else
#endif
{
hasFastPools = gcvTRUE;
status = gckVIDMEM_AllocateLinear(Kernel,
videoMemory,
Bytes,
Alignment,
Type,
(*Pool == gcvPOOL_SYSTEM),
&node);
}
if (gcmIS_SUCCESS(status))
{
/* Memory allocated. */
node->VidMem.pool = pool;
bytes = node->VidMem.bytes;
break;
}
}
}
if (pool == gcvPOOL_LOCAL_INTERNAL)
{
/* Advance to external memory. */
pool = gcvPOOL_LOCAL_EXTERNAL;
}
else
if (pool == gcvPOOL_LOCAL_EXTERNAL)
{
/* Advance to contiguous system memory. */
pool = gcvPOOL_SYSTEM;
}
else
if (pool == gcvPOOL_SYSTEM)
{
/* Do not go ahead to try relative slow pools */
if (fastPools && hasFastPools)
{
status = gcvSTATUS_OUT_OF_MEMORY;
break;
}
/* Advance to contiguous memory. */
pool = gcvPOOL_CONTIGUOUS;
}
else
if (pool == gcvPOOL_CONTIGUOUS)
{
#if gcdENABLE_VG
if (Kernel->vg)
{
tileStatusInVirtual = gcvFALSE;
}
else
#endif
{
tileStatusInVirtual =
gckHARDWARE_IsFeatureAvailable(Kernel->hardware,
gcvFEATURE_MC20);
}
if (Type == gcvSURF_TILE_STATUS && tileStatusInVirtual != gcvTRUE)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
if (contiguous)
{
break;
}
/* Advance to virtual memory. */
pool = gcvPOOL_VIRTUAL;
}
else
{
/* Out of pools. */
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
}
if (node == gcvNULL)
{
if (contiguous)
{
/* Broadcast OOM message. */
status = gckOS_Broadcast(Kernel->os, Kernel->hardware, gcvBROADCAST_OUT_OF_MEMORY);
if (gcmIS_SUCCESS(status))
{
/* Get some memory. */
gckOS_Delay(gcvNULL, 1);
goto AllocateMemory;
}
}
/* Nothing allocated. */
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
/* Allocate handle for this video memory. */
gcmkONERROR(
gckVIDMEM_NODE_Allocate(Kernel, node, Type, pool, &handle));
/* Return node and pool used for allocation. */
*Node = handle;
*Pool = pool;
/* Encode surface type and pool to database type. */
type = gcvDB_VIDEO_MEMORY
| (Type << gcdDB_VIDEO_MEMORY_TYPE_SHIFT)
| (pool << gcdDB_VIDEO_MEMORY_POOL_SHIFT);
/* Record in process db. */
gcmkONERROR(
gckKERNEL_AddProcessDB(Kernel,
ProcessID,
type,
gcmINT2PTR(handle),
gcvNULL,
bytes));
/* Return status. */
gcmkFOOTER_ARG("*Pool=%d *Node=0x%x", *Pool, *Node);
return gcvSTATUS_OK;
OnError:
if (handle)
{
/* Destroy handle allocated. */
gcmkVERIFY_OK(gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, handle));
}
if (node)
{
/* Free video memory allocated. */
gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_ReleaseVideoMemory
**
** Release handle of a video memory.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctUINT32 ProcessID
** ProcessID of current process.
**
** gctUINT32 Handle
** Handle of video memory.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckKERNEL_ReleaseVideoMemory(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Handle
)
{
gceSTATUS status;
gckVIDMEM_NODE nodeObject;
gceDATABASE_TYPE type;
gcmkHEADER_ARG("Kernel=0x%08X ProcessID=%d Handle=%d",
Kernel, ProcessID, Handle);
gcmkONERROR(
gckVIDMEM_HANDLE_Lookup(Kernel, ProcessID, Handle, &nodeObject));
type = gcvDB_VIDEO_MEMORY
| (nodeObject->type << gcdDB_VIDEO_MEMORY_TYPE_SHIFT)
| (nodeObject->pool << gcdDB_VIDEO_MEMORY_POOL_SHIFT);
gcmkONERROR(
gckKERNEL_RemoveProcessDB(Kernel,
ProcessID,
type,
gcmINT2PTR(Handle)));
gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, Handle);
gckVIDMEM_NODE_Dereference(Kernel, nodeObject);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_LockVideoMemory
**
** Lock a video memory node. It will generate a cpu virtual address used
** by software and a GPU address used by GPU.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gceCORE Core
** GPU to which video memory is locked.
**
** gcsHAL_INTERFACE * Interface
** Pointer to a gcsHAL_INTERFACE structure that defines the command to
** be dispatched.
**
** OUTPUT:
**
** gcsHAL_INTERFACE * Interface
** Pointer to a gcsHAL_INTERFACE structure that receives any data to be
** returned.
*/
gceSTATUS
gckKERNEL_LockVideoMemory(
IN gckKERNEL Kernel,
IN gceCORE Core,
IN gctUINT32 ProcessID,
IN gctBOOL FromUser,
IN OUT gcsHAL_INTERFACE * Interface
)
{
gceSTATUS status;
gckVIDMEM_NODE nodeObject = gcvNULL;
gcuVIDMEM_NODE_PTR node = gcvNULL;
gctBOOL locked = gcvFALSE;
gctBOOL asynchronous = gcvFALSE;
#ifndef __QNXNTO__
gctPOINTER pointer = gcvNULL;
#endif
gcmkHEADER_ARG("Kernel=0x%08X ProcessID=%d",
Kernel, ProcessID);
gcmkONERROR(
gckVIDMEM_HANDLE_LookupAndReference(Kernel,
Interface->u.LockVideoMemory.node,
&nodeObject));
node = nodeObject->node;
Interface->u.LockVideoMemory.gid = 0;
/* Lock video memory. */
gcmkONERROR(
gckVIDMEM_Lock(Kernel,
nodeObject,
Interface->u.LockVideoMemory.cacheable,
&Interface->u.LockVideoMemory.address,
&Interface->u.LockVideoMemory.gid,
&Interface->u.LockVideoMemory.physicalAddress));
locked = gcvTRUE;
if (node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
{
/* Map video memory address into user space. */
#ifdef __QNXNTO__
if (node->VidMem.logical == gcvNULL)
{
gcmkONERROR(
gckKERNEL_MapVideoMemory(Kernel,
FromUser,
Interface->u.LockVideoMemory.address,
ProcessID,
node->VidMem.bytes,
&node->VidMem.logical));
}
gcmkASSERT(node->VidMem.logical != gcvNULL);
Interface->u.LockVideoMemory.memory = gcmPTR_TO_UINT64(node->VidMem.logical);
#else
gcmkONERROR(
gckKERNEL_MapVideoMemoryEx(Kernel,
Core,
FromUser,
Interface->u.LockVideoMemory.address,
node->VidMem.pool,
&pointer));
Interface->u.LockVideoMemory.memory = gcmPTR_TO_UINT64(pointer);
#endif
}
else
{
Interface->u.LockVideoMemory.memory = gcmPTR_TO_UINT64(node->Virtual.logical);
/* Success. */
status = gcvSTATUS_OK;
}
#if gcdPROCESS_ADDRESS_SPACE
gcmkONERROR(gckVIDMEM_Node_Lock(
Kernel,
nodeObject,
&Interface->u.LockVideoMemory.address
));
#endif
#if gcdSECURE_USER
/* Return logical address as physical address. */
Interface->u.LockVideoMemory.address =
(gctUINT32)(Interface->u.LockVideoMemory.memory);
#endif
gcmkONERROR(
gckKERNEL_AddProcessDB(Kernel,
ProcessID, gcvDB_VIDEO_MEMORY_LOCKED,
gcmINT2PTR(Interface->u.LockVideoMemory.node),
gcvNULL,
0));
gckVIDMEM_HANDLE_Reference(
Kernel, ProcessID, (gctUINT32)Interface->u.LockVideoMemory.node);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (locked)
{
/* Roll back the lock. */
gcmkVERIFY_OK(gckVIDMEM_Unlock(Kernel,
nodeObject,
gcvSURF_TYPE_UNKNOWN,
&asynchronous));
if (gcvTRUE == asynchronous)
{
/* Bottom Half */
gcmkVERIFY_OK(gckVIDMEM_Unlock(Kernel,
nodeObject,
gcvSURF_TYPE_UNKNOWN,
gcvNULL));
}
}
if (nodeObject != gcvNULL)
{
gckVIDMEM_NODE_Dereference(Kernel, nodeObject);
}
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_UnlockVideoMemory
**
** Unlock a video memory node.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctUINT32 ProcessID
** ProcessID of current process.
**
** gcsHAL_INTERFACE * Interface
** Pointer to a gcsHAL_INTERFACE structure that defines the command to
** be dispatched.
**
** OUTPUT:
**
** gcsHAL_INTERFACE * Interface
** Pointer to a gcsHAL_INTERFACE structure that receives any data to be
** returned.
*/
gceSTATUS
gckKERNEL_UnlockVideoMemory(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN OUT gcsHAL_INTERFACE * Interface
)
{
gceSTATUS status;
gckVIDMEM_NODE nodeObject;
gcuVIDMEM_NODE_PTR node;
gctSIZE_T bytes;
gcmkHEADER_ARG("Kernel=0x%08X ProcessID=%d",
Kernel, ProcessID);
Interface->u.UnlockVideoMemory.pool = gcvPOOL_UNKNOWN;
Interface->u.UnlockVideoMemory.bytes = 0;
gcmkONERROR(gckVIDMEM_HANDLE_Lookup(
Kernel,
ProcessID,
(gctUINT32)Interface->u.UnlockVideoMemory.node,
&nodeObject));
node = nodeObject->node;
bytes = (node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
? node->VidMem.bytes
: node->Virtual.bytes;
/* Unlock video memory. */
gcmkONERROR(gckVIDMEM_Unlock(
Kernel,
nodeObject,
Interface->u.UnlockVideoMemory.type,
&Interface->u.UnlockVideoMemory.asynchroneous));
#if gcdSECURE_USER
/* Flush the translation cache for virtual surfaces. */
if (node->VidMem.memory->object.type != gcvOBJ_VIDMEM)
{
gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(Kernel,
cache,
node->Virtual.logical,
bytes));
}
#endif
Interface->u.UnlockVideoMemory.pool = nodeObject->pool;
Interface->u.UnlockVideoMemory.bytes = (gctUINT)bytes;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_BottomHalfUnlockVideoMemory
**
** Unlock video memory from gpu.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctUINT32 ProcessID
** Process ID owning this memory.
**
** gctPOINTER Pointer
** Video memory to be unlock.
*/
gceSTATUS
gckKERNEL_BottomHalfUnlockVideoMemory(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Node
)
{
gceSTATUS status;
gckVIDMEM_NODE nodeObject = gcvNULL;
/* Remove record from process db. */
gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
Kernel,
ProcessID,
gcvDB_VIDEO_MEMORY_LOCKED,
gcmINT2PTR(Node)));
gcmkONERROR(gckVIDMEM_HANDLE_Lookup(
Kernel,
ProcessID,
Node,
&nodeObject));
gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, Node);
/* Unlock video memory. */
gcmkONERROR(gckVIDMEM_Unlock(
Kernel,
nodeObject,
gcvSURF_TYPE_UNKNOWN,
gcvNULL));
gcmkONERROR(gckVIDMEM_NODE_Dereference(
Kernel,
nodeObject));
return gcvSTATUS_OK;
OnError:
return status;
}
/*******************************************************************************
**
** gckKERNEL_SetVidMemMetadata
**
** Set/Get metadata to/from gckVIDMEM_NODE object.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctUINT32 ProcessID
** ProcessID of current process.
**
** INOUT:
**
** gcsHAL_INTERFACE * Interface
** Pointer to a interface structure
*/
#if defined(CONFIG_DMA_SHARED_BUFFER)
#include <linux/dma-buf.h>
gceSTATUS
gckKERNEL_SetVidMemMetadata(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
INOUT gcsHAL_INTERFACE * Interface
)
{
gceSTATUS status = gcvSTATUS_NOT_SUPPORTED;
gckVIDMEM_NODE nodeObj = gcvNULL;
gcmkHEADER_ARG("Kernel=0x%X ProcessID=%d", Kernel, ProcessID);
gcmkONERROR(gckVIDMEM_HANDLE_Lookup(Kernel, ProcessID, Interface->u.SetVidMemMetadata.node, &nodeObj));
if (Interface->u.SetVidMemMetadata.readback)
{
Interface->u.SetVidMemMetadata.ts_fd = nodeObj->metadata.ts_fd;
Interface->u.SetVidMemMetadata.fc_enabled = nodeObj->metadata.fc_enabled;
Interface->u.SetVidMemMetadata.fc_value = nodeObj->metadata.fc_value;
Interface->u.SetVidMemMetadata.fc_value_upper = nodeObj->metadata.fc_value_upper;
Interface->u.SetVidMemMetadata.compressed = nodeObj->metadata.compressed;
Interface->u.SetVidMemMetadata.compress_format = nodeObj->metadata.compress_format;
}
else
{
#ifdef gcdANDROID
if (nodeObj->metadata.ts_address == 0 && nodeObj->tsNode != NULL)
{
gctUINT32 Address = 0;
gctUINT32 Gid = 0;
gctUINT64 PhysicalAddress = 0;
gcmkONERROR(gckVIDMEM_Lock(Kernel,
nodeObj->tsNode,
gcvFALSE,
&Address,
&Gid,
&PhysicalAddress));
nodeObj->metadata.ts_address = (
PhysicalAddress + Kernel->hardware->baseAddress);
gcmkONERROR(gckVIDMEM_Unlock(Kernel,
nodeObj->tsNode,
gcvSURF_TYPE_UNKNOWN,
gcvNULL));
}
#else
nodeObj->metadata.ts_fd = Interface->u.SetVidMemMetadata.ts_fd;
if (nodeObj->metadata.ts_fd >= 0)
{
nodeObj->metadata.ts_dma_buf = dma_buf_get(nodeObj->metadata.ts_fd);
if (IS_ERR(nodeObj->metadata.ts_dma_buf))
{
gcmkONERROR(gcvSTATUS_NOT_FOUND);
}
dma_buf_put(nodeObj->metadata.ts_dma_buf);
}
else
{
nodeObj->metadata.ts_dma_buf = NULL;
}
#endif
nodeObj->metadata.fc_enabled = Interface->u.SetVidMemMetadata.fc_enabled;
nodeObj->metadata.fc_value = Interface->u.SetVidMemMetadata.fc_value;
nodeObj->metadata.fc_value_upper = Interface->u.SetVidMemMetadata.fc_value_upper;
nodeObj->metadata.compressed = Interface->u.SetVidMemMetadata.compressed;
nodeObj->metadata.compress_format = Interface->u.SetVidMemMetadata.compress_format;
}
OnError:
gcmkFOOTER();
return status;
}
#else
gceSTATUS
gckKERNEL_SetVidMemMetadata(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
INOUT gcsHAL_INTERFACE * Interface
)
{
gcmkFATAL("The kernel did NOT support CONFIG_DMA_SHARED_BUFFER");
return gcvSTATUS_NOT_SUPPORTED;
}
#endif
/*******************************************************************************
**
** gckKERNEL_QueryVidMemPoolNodes
**
** Loop all databases to query used memory nodes of a specific pool.
**
** INPUT:
**
** Pool The memory pool for query.
** TotalSize Total size of the used contiguous memory.
**
** MemoryBlocks Used contiguous memory block info.
**
** NumMaxBlock The count of memory blocks array provided by the caller.
**
** NumBlocks The actual number of memory blocks returned.
**
** OUTPUT:
**
** Error status. Should always be gcvSTATUS_SUCCESS since a query should always succeed.
*/
gceSTATUS
gckKERNEL_QueryVidMemPoolNodes(
gckKERNEL Kernel,
gcePOOL Pool,
gctUINT32 * TotalSize,
gcsContiguousBlock * MemoryBlocks,
gctUINT32 NumMaxBlocks,
gctUINT32 * NumBlocks
)
{
gceSTATUS status = gcvSTATUS_OK;
gctINT i;
gctINT bcount;
gctUINT32 num_blocks = 0;
gctSIZE_T total_size = 0;
gckVIDMEM memory;
gcuVIDMEM_NODE_PTR nodes, node;
do
{
/* Get the heap and nodes. */
status = gckKERNEL_GetVideoMemoryPool(Kernel, Pool, &memory);
if (status != gcvSTATUS_OK)
break;
status = gckVIDMEM_QueryNodes(Kernel, Pool, &bcount, &nodes);
if (status != gcvSTATUS_OK)
break;
/* Iterate all nodes. */
for (i = 0; i < bcount; i++)
{
node = nodes[i].VidMem.next;
do
{
if (node == gcvNULL)
{
break;
}
/* Is it in the "free" list? */
if (node->VidMem.nextFree == gcvNULL)
{
if (num_blocks < NumMaxBlocks)
{
MemoryBlocks[num_blocks].ptr = (gctUINT32)node->VidMem.offset + memory->baseAddress;
MemoryBlocks[num_blocks].size = node->VidMem.bytes;
}
total_size += node->VidMem.bytes;
num_blocks++;
}
node = node->VidMem.next;
} while (node != &nodes[i]);
}
}
while (gcvFALSE);
if (TotalSize != gcvNULL)
*TotalSize = (gctUINT32)total_size;
if (NumBlocks != gcvNULL)
*NumBlocks = num_blocks;
return status;
}
gceSTATUS
gckKERNEL_QueryDatabase(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN OUT gcsHAL_INTERFACE * Interface
)
{
gceSTATUS status;
gctINT i;
gceDATABASE_TYPE type[3] = {
gcvDB_VIDEO_MEMORY | (gcvPOOL_SYSTEM << gcdDB_VIDEO_MEMORY_POOL_SHIFT),
gcvDB_VIDEO_MEMORY | (gcvPOOL_CONTIGUOUS << gcdDB_VIDEO_MEMORY_POOL_SHIFT),
gcvDB_VIDEO_MEMORY | (gcvPOOL_VIRTUAL << gcdDB_VIDEO_MEMORY_POOL_SHIFT),
};
gcmkHEADER();
/* Query video memory. */
gcmkONERROR(
gckKERNEL_QueryProcessDB(Kernel,
Interface->u.Database.processID,
!Interface->u.Database.validProcessID,
gcvDB_VIDEO_MEMORY,
&Interface->u.Database.vidMem));
/* Query non-paged memory. */
gcmkONERROR(
gckKERNEL_QueryProcessDB(Kernel,
Interface->u.Database.processID,
!Interface->u.Database.validProcessID,
gcvDB_NON_PAGED,
&Interface->u.Database.nonPaged));
/* Query contiguous memory. */
gcmkONERROR(
gckKERNEL_QueryProcessDB(Kernel,
Interface->u.Database.processID,
!Interface->u.Database.validProcessID,
gcvDB_CONTIGUOUS,
&Interface->u.Database.contiguous));
/* Query GPU idle time. */
gcmkONERROR(
gckKERNEL_QueryProcessDB(Kernel,
Interface->u.Database.processID,
!Interface->u.Database.validProcessID,
gcvDB_IDLE,
&Interface->u.Database.gpuIdle));
for (i = 0; i < 3; i++)
{
/* Query each video memory pool. */
gcmkONERROR(
gckKERNEL_QueryProcessDB(Kernel,
Interface->u.Database.processID,
!Interface->u.Database.validProcessID,
type[i],
&Interface->u.Database.vidMemPool[i]));
}
#if gcmIS_DEBUG(gcdDEBUG_TRACE)
gckKERNEL_DumpVidMemUsage(Kernel, Interface->u.Database.processID);
#endif
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
gceSTATUS
gckKERNEL_ConfigPowerManagement(
IN gckKERNEL Kernel,
IN OUT gcsHAL_INTERFACE * Interface
)
{
gceSTATUS status;
gctBOOL enable = Interface->u.ConfigPowerManagement.enable;
gcmkHEADER();
gcmkONERROR(gckHARDWARE_SetPowerManagement(Kernel->hardware, enable));
if (enable == gcvFALSE)
{
gcmkONERROR(
gckHARDWARE_SetPowerManagementState(Kernel->hardware, gcvPOWER_ON));
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkFOOTER();
return status;
}
static gceSTATUS
gckKERNEL_CacheOperation(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Node,
IN gceCACHEOPERATION Operation,
IN gctPOINTER Logical,
IN gctSIZE_T Bytes
)
{
gceSTATUS status;
gckVIDMEM_NODE nodeObject = gcvNULL;
gcuVIDMEM_NODE_PTR node = gcvNULL;
void *memHandle;
gcmkHEADER_ARG("Kernel=%p pid=%u Node=%u op=%d Logical=%p Bytes=0x%lx",
Kernel, ProcessID, Node, Operation, Logical, Bytes);
gcmkONERROR(gckVIDMEM_HANDLE_Lookup(Kernel,
ProcessID,
Node,
&nodeObject));
node = nodeObject->node;
if (node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
{
static gctBOOL printed;
if (!printed)
{
printed = gcvTRUE;
gcmkPRINT("[galcore]: %s: Flush Video Memory", __FUNCTION__);
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
else
{
memHandle = node->Virtual.physical;
}
switch (Operation)
{
case gcvCACHE_FLUSH:
/* Clean and invalidate the cache. */
status = gckOS_CacheFlush(Kernel->os,
ProcessID,
memHandle,
gcvINVALID_PHYSICAL_ADDRESS,
Logical,
Bytes);
break;
case gcvCACHE_CLEAN:
/* Clean the cache. */
status = gckOS_CacheClean(Kernel->os,
ProcessID,
memHandle,
gcvINVALID_PHYSICAL_ADDRESS,
Logical,
Bytes);
break;
case gcvCACHE_INVALIDATE:
/* Invalidate the cache. */
status = gckOS_CacheInvalidate(Kernel->os,
ProcessID,
memHandle,
gcvINVALID_PHYSICAL_ADDRESS,
Logical,
Bytes);
break;
case gcvCACHE_MEMORY_BARRIER:
status = gckOS_MemoryBarrier(Kernel->os, Logical);
break;
default:
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
break;
}
OnError:
gcmkFOOTER();
return status;
}
gceSTATUS
gckKERNEL_WaitFence(
IN gckKERNEL Kernel,
IN gctUINT32 Handle,
IN gctUINT32 TimeOut
)
{
gceSTATUS status;
gckVIDMEM_NODE node;
gctUINT32 processID;
gckCOMMAND command = Kernel->command;
gckASYNC_COMMAND asyncCommand = Kernel->asyncCommand;
gckFENCE fence = gcvNULL;
gctUINT i;
gckOS_GetProcessID(&processID);
gcmkONERROR(gckVIDMEM_HANDLE_LookupAndReference(Kernel, Handle, &node));
/* Wait for fence of all engines. */
for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++)
{
gckFENCE_SYNC sync = &node->sync[i];
if (i == gcvENGINE_RENDER)
{
fence = command->fence;
}
else
{
fence = asyncCommand->fence;
}
#if USE_KERNEL_VIRTUAL_BUFFERS
if (Kernel->virtualCommandBuffer)
{
gckVIRTUAL_COMMAND_BUFFER_PTR commandBuffer = (gckVIRTUAL_COMMAND_BUFFER_PTR) fence->physical;
fence->physHandle = commandBuffer->virtualBuffer.physical;
}
else
#endif
{
fence->physHandle = fence->physical;
}
gcmkONERROR(gckOS_CacheInvalidate(
Kernel->os,
0,
fence->physHandle,
0,
fence->logical,
8
));
if (sync->commitStamp <= *(gctUINT64_PTR)fence->logical)
{
continue;
}
else
{
gckOS_Signal(Kernel->os, sync->signal, gcvFALSE);
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, &fence->mutex, gcvINFINITE));
/* Add to waiting list. */
gcsLIST_AddTail(&sync->head, &fence->waitingList);
gcmkASSERT(sync->inList == gcvFALSE);
sync->inList = gcvTRUE;
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, &fence->mutex));
/* Wait. */
status = gckOS_WaitSignal(Kernel->os, sync->signal, gcvTRUE, TimeOut);
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, &fence->mutex, gcvINFINITE));
if (sync->inList)
{
gcsLIST_Del(&sync->head);
sync->inList = gcvFALSE;
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, &fence->mutex));
}
}
gckVIDMEM_NODE_Dereference(Kernel, node);
OnError:
return status;
}
#ifdef __linux__
typedef struct _gcsGRRAPHIC_BUFFER_PARCLE
{
gcsFDPRIVATE base;
gckKERNEL kernel;
gckVIDMEM_NODE node[3];
gctSHBUF shBuf;
gctINT32 signal;
}
gcsGRAPHIC_BUFFER_PARCLE;
static void
_ReleaseGraphicBuffer(
gckKERNEL Kernel,
gcsGRAPHIC_BUFFER_PARCLE * Parcle
)
{
gctUINT i;
for (i = 0; i < 3; i++)
{
if (Parcle->node[i])
{
gckVIDMEM_NODE_Dereference(Kernel, Parcle->node[i]);
}
}
if (Parcle->shBuf)
{
gckKERNEL_DestroyShBuffer(Kernel, Parcle->shBuf);
}
if (Parcle->signal)
{
gckOS_DestroyUserSignal(Kernel->os, Parcle->signal);
}
gcmkOS_SAFE_FREE(Kernel->os, Parcle);
}
static gctINT
_FdReleaseGraphicBuffer(
gcsFDPRIVATE_PTR Private
)
{
gcsGRAPHIC_BUFFER_PARCLE * parcle = (gcsGRAPHIC_BUFFER_PARCLE *) Private;
_ReleaseGraphicBuffer(parcle->kernel, parcle);
return 0;
}
static gceSTATUS
_GetGraphicBufferFd(
IN gckKERNEL Kernel,
IN gctUINT32 ProcessID,
IN gctUINT32 Node[3],
IN gctUINT64 ShBuf,
IN gctUINT64 Signal,
OUT gctINT32 * Fd
)
{
gceSTATUS status;
gctUINT i;
gcsGRAPHIC_BUFFER_PARCLE * parcle = gcvNULL;
gcmkONERROR(gckOS_Allocate(
Kernel->os,
gcmSIZEOF(gcsGRAPHIC_BUFFER_PARCLE),
(gctPOINTER *)&parcle
));
gckOS_ZeroMemory(parcle, sizeof(gcsGRAPHIC_BUFFER_PARCLE));
parcle->base.release = _FdReleaseGraphicBuffer;
parcle->kernel = Kernel;
for (i = 0; i < 3; i++)
{
if (Node[i] != 0)
{
gcmkONERROR(
gckVIDMEM_HANDLE_LookupAndReference(Kernel, Node[i], &parcle->node[i]));
}
}
if (ShBuf)
{
gctSHBUF shBuf = gcmUINT64_TO_PTR(ShBuf);
gcmkONERROR(gckKERNEL_MapShBuffer(Kernel, shBuf));
parcle->shBuf = shBuf;
}
if (Signal)
{
gctSIGNAL signal = gcmUINT64_TO_PTR(Signal);
gcmkONERROR(
gckOS_MapSignal(Kernel->os,
signal,
(gctHANDLE)(gctUINTPTR_T)ProcessID,
&signal));
parcle->signal= (gctINT32)Signal;
}
gcmkONERROR(gckOS_GetFd("viv-gr", &parcle->base, Fd));
return gcvSTATUS_OK;
OnError:
if (parcle)
{
_ReleaseGraphicBuffer(Kernel, parcle);
}
return status;
}
#endif
/*******************************************************************************
**
** gckKERNEL_Dispatch
**
** Dispatch a command received from the user HAL layer.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctBOOL FromUser
** whether the call is from the user space.
**
** gcsHAL_INTERFACE * Interface
** Pointer to a gcsHAL_INTERFACE structure that defines the command to
** be dispatched.
**
** OUTPUT:
**
** gcsHAL_INTERFACE * Interface
** Pointer to a gcsHAL_INTERFACE structure that receives any data to be
** returned.
*/
gceSTATUS
gckKERNEL_Dispatch(
IN gckKERNEL Kernel,
IN gckDEVICE Device,
IN gctBOOL FromUser,
IN OUT gcsHAL_INTERFACE * Interface
)
{
gceSTATUS status = gcvSTATUS_OK;
gctPHYS_ADDR physical = gcvNULL;
gctSIZE_T bytes;
gctPOINTER logical = gcvNULL;
#if (gcdENABLE_3D || gcdENABLE_2D)
gckCONTEXT context = gcvNULL;
#endif
gckKERNEL kernel = Kernel;
gctUINT32 processID;
#if gcdSECURE_USER
gcskSECURE_CACHE_PTR cache;
gctPOINTER logical;
#endif
#if !USE_NEW_LINUX_SIGNAL
gctSIGNAL signal;
#endif
gckVIRTUAL_COMMAND_BUFFER_PTR buffer;
gctBOOL powerMutexAcquired = gcvFALSE;
gctBOOL commitMutexAcquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%x FromUser=%d Interface=0x%x",
Kernel, FromUser, Interface);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(Interface != gcvNULL);
#if gcmIS_DEBUG(gcdDEBUG_TRACE)
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL,
"Dispatching command %d (%s)",
Interface->command, _DispatchText[Interface->command]);
#endif
#if QNX_SINGLE_THREADED_DEBUGGING
gckOS_AcquireMutex(Kernel->os, Kernel->debugMutex, gcvINFINITE);
#endif
/* Get the current process ID. */
gcmkONERROR(gckOS_GetProcessID(&processID));
#if gcdSECURE_USER
gcmkONERROR(gckKERNEL_GetProcessDBCache(Kernel, processID, &cache));
#endif
/* Dispatch on command. */
switch (Interface->command)
{
case gcvHAL_GET_BASE_ADDRESS:
/* Get base address. */
Interface->u.GetBaseAddress.baseAddress = Kernel->hardware->baseAddress;
Interface->u.GetBaseAddress.flatMappingRangeCount = Kernel->mmu->flatMappingRangeCount;
if (Kernel->mmu->flatMappingRangeCount)
{
gckOS_MemCopy(Interface->u.GetBaseAddress.flatMappingRanges, Kernel->mmu->flatMappingRanges,
gcmSIZEOF(gcsFLAT_MAPPING_RANGE) * Kernel->mmu->flatMappingRangeCount);
}
break;
case gcvHAL_QUERY_VIDEO_MEMORY:
/* Query video memory size. */
gcmkONERROR(gckKERNEL_QueryVideoMemory(Kernel, Interface));
break;
case gcvHAL_QUERY_CHIP_IDENTITY:
/* Query chip identity. */
gcmkONERROR(
gckHARDWARE_QueryChipIdentity(
Kernel->hardware,
&Interface->u.QueryChipIdentity));
break;
case gcvHAL_QUERY_CHIP_FREQUENCY:
/* Query chip clock. */
gcmkONERROR(
gckHARDWARE_QueryFrequency(Kernel->hardware));
Interface->u.QueryChipFrequency.mcClk = Kernel->hardware->mcClk;
Interface->u.QueryChipFrequency.shClk = Kernel->hardware->shClk;
break;
case gcvHAL_MAP_MEMORY:
physical = gcmINT2PTR(Interface->u.MapMemory.physical);
/* Map memory. */
gcmkONERROR(
gckKERNEL_MapMemory(Kernel,
physical,
(gctSIZE_T) Interface->u.MapMemory.bytes,
&logical));
Interface->u.MapMemory.logical = gcmPTR_TO_UINT64(logical);
gcmkVERIFY_OK(
gckKERNEL_AddProcessDB(Kernel,
processID, gcvDB_MAP_MEMORY,
logical,
physical,
(gctSIZE_T) Interface->u.MapMemory.bytes));
break;
case gcvHAL_UNMAP_MEMORY:
physical = gcmINT2PTR(Interface->u.UnmapMemory.physical);
gcmkVERIFY_OK(
gckKERNEL_RemoveProcessDB(Kernel,
processID, gcvDB_MAP_MEMORY,
gcmUINT64_TO_PTR(Interface->u.UnmapMemory.logical)));
/* Unmap memory. */
gcmkONERROR(
gckKERNEL_UnmapMemory(Kernel,
physical,
(gctSIZE_T) Interface->u.UnmapMemory.bytes,
gcmUINT64_TO_PTR(Interface->u.UnmapMemory.logical),
processID));
break;
case gcvHAL_ALLOCATE_NON_PAGED_MEMORY:
bytes = (gctSIZE_T) Interface->u.AllocateNonPagedMemory.bytes;
/* Allocate non-paged memory. */
gcmkONERROR(
gckOS_AllocateNonPagedMemory(
Kernel->os,
FromUser,
gcvALLOC_FLAG_CONTIGUOUS,
&bytes,
&physical,
&logical));
Interface->u.AllocateNonPagedMemory.bytes = bytes;
Interface->u.AllocateNonPagedMemory.logical = gcmPTR_TO_UINT64(logical);
Interface->u.AllocateNonPagedMemory.physical = gcmPTR_TO_NAME(physical);
gcmkVERIFY_OK(
gckKERNEL_AddProcessDB(Kernel,
processID, gcvDB_NON_PAGED,
logical,
gcmINT2PTR(Interface->u.AllocateNonPagedMemory.physical),
bytes));
break;
case gcvHAL_ALLOCATE_VIRTUAL_COMMAND_BUFFER:
bytes = (gctSIZE_T) Interface->u.AllocateVirtualCommandBuffer.bytes;
gcmkONERROR(
gckKERNEL_AllocateVirtualCommandBuffer(
Kernel,
FromUser,
&bytes,
&physical,
&logical));
Interface->u.AllocateVirtualCommandBuffer.bytes = bytes;
Interface->u.AllocateVirtualCommandBuffer.logical = gcmPTR_TO_UINT64(logical);
Interface->u.AllocateVirtualCommandBuffer.physical = gcmPTR_TO_NAME(physical);
gcmkVERIFY_OK(
gckKERNEL_AddProcessDB(Kernel,
processID, gcvDB_COMMAND_BUFFER,
logical,
gcmINT2PTR(Interface->u.AllocateVirtualCommandBuffer.physical),
bytes));
break;
case gcvHAL_FREE_NON_PAGED_MEMORY:
physical = gcmNAME_TO_PTR(Interface->u.FreeNonPagedMemory.physical);
gcmkVERIFY_OK(
gckKERNEL_RemoveProcessDB(Kernel,
processID, gcvDB_NON_PAGED,
gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
/* Unmap user logical out of physical memory first. */
gcmkONERROR(gckOS_UnmapUserLogical(Kernel->os,
physical,
(gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes,
gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
/* Free non-paged memory. */
gcmkONERROR(
gckOS_FreeNonPagedMemory(Kernel->os,
(gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes,
physical,
gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
#if gcdSECURE_USER
gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
Kernel,
cache,
gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical),
(gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes));
#endif
gcmRELEASE_NAME(Interface->u.FreeNonPagedMemory.physical);
break;
case gcvHAL_ALLOCATE_CONTIGUOUS_MEMORY:
bytes = (gctSIZE_T) Interface->u.AllocateContiguousMemory.bytes;
/* Allocate contiguous memory. */
gcmkONERROR(gckOS_AllocateContiguous(
Kernel->os,
FromUser,
&bytes,
&physical,
&logical));
Interface->u.AllocateContiguousMemory.bytes = bytes;
Interface->u.AllocateContiguousMemory.logical = gcmPTR_TO_UINT64(logical);
Interface->u.AllocateContiguousMemory.physical = gcmPTR_TO_NAME(physical);
gcmkONERROR(gckHARDWARE_ConvertLogical(
Kernel->hardware,
logical,
gcvTRUE,
&Interface->u.AllocateContiguousMemory.address));
gcmkVERIFY_OK(gckKERNEL_AddProcessDB(
Kernel,
processID, gcvDB_CONTIGUOUS,
logical,
gcmINT2PTR(Interface->u.AllocateContiguousMemory.physical),
bytes));
break;
case gcvHAL_FREE_CONTIGUOUS_MEMORY:
physical = gcmNAME_TO_PTR(Interface->u.FreeContiguousMemory.physical);
gcmkVERIFY_OK(
gckKERNEL_RemoveProcessDB(Kernel,
processID, gcvDB_CONTIGUOUS,
gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
/* Unmap user logical out of physical memory first. */
gcmkONERROR(gckOS_UnmapUserLogical(Kernel->os,
physical,
(gctSIZE_T) Interface->u.FreeContiguousMemory.bytes,
gcmUINT64_TO_PTR(Interface->u.FreeContiguousMemory.logical)));
/* Free contiguous memory. */
gcmkONERROR(
gckOS_FreeContiguous(Kernel->os,
physical,
gcmUINT64_TO_PTR(Interface->u.FreeContiguousMemory.logical),
(gctSIZE_T) Interface->u.FreeContiguousMemory.bytes));
#if gcdSECURE_USER
gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
Kernel,
cache,
gcmUINT64_TO_PTR(Interface->u.FreeContiguousMemory.logical),
(gctSIZE_T) Interface->u.FreeContiguousMemory.bytes));
#endif
gcmRELEASE_NAME(Interface->u.FreeContiguousMemory.physical);
break;
case gcvHAL_ALLOCATE_VIDEO_MEMORY:
gcmkONERROR(gcvSTATUS_NOT_SUPPORTED);
break;
case gcvHAL_ALLOCATE_LINEAR_VIDEO_MEMORY:
/* Allocate memory. */
gcmkONERROR(
gckKERNEL_AllocateLinearMemory(Kernel, processID,
&Interface->u.AllocateLinearVideoMemory.pool,
Interface->u.AllocateLinearVideoMemory.bytes,
Interface->u.AllocateLinearVideoMemory.alignment,
Interface->u.AllocateLinearVideoMemory.type,
Interface->u.AllocateLinearVideoMemory.flag,
&Interface->u.AllocateLinearVideoMemory.node));
break;
case gcvHAL_RELEASE_VIDEO_MEMORY:
/* Release video memory. */
gcmkONERROR(gckKERNEL_ReleaseVideoMemory(
Kernel, processID,
(gctUINT32)Interface->u.ReleaseVideoMemory.node
));
break;
case gcvHAL_LOCK_VIDEO_MEMORY:
/* Lock video memory. */
gcmkONERROR(gckKERNEL_LockVideoMemory(Kernel, Kernel->core, processID, FromUser, Interface));
break;
case gcvHAL_UNLOCK_VIDEO_MEMORY:
/* Unlock video memory. */
gcmkONERROR(gckKERNEL_UnlockVideoMemory(Kernel, processID, Interface));
break;
case gcvHAL_BOTTOM_HALF_UNLOCK_VIDEO_MEMORY:
gcmkERR_BREAK(gckKERNEL_BottomHalfUnlockVideoMemory(Kernel, processID,
Interface->u.BottomHalfUnlockVideoMemory.node));
break;
case gcvHAL_EVENT_COMMIT:
if (!Interface->commitMutex)
{
gcmkONERROR(gckOS_AcquireMutex(Kernel->os,
Kernel->device->commitMutex,
gcvINFINITE
));
commitMutexAcquired = gcvTRUE;
}
/* Commit an event queue. */
if (Interface->engine == gcvENGINE_BLT)
{
if (!gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_ASYNC_BLIT))
{
gcmkONERROR(gcvSTATUS_NOT_SUPPORTED);
}
gcmkONERROR(gckEVENT_Commit(
Kernel->asyncEvent, gcmUINT64_TO_PTR(Interface->u.Event.queue), gcvFALSE));
}
else
{
gcmkONERROR(gckEVENT_Commit(
Kernel->eventObj, gcmUINT64_TO_PTR(Interface->u.Event.queue), gcvFALSE));
}
if (!Interface->commitMutex)
{
gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->device->commitMutex));
commitMutexAcquired = gcvFALSE;
}
break;
case gcvHAL_COMMIT:
if (!Interface->commitMutex)
{
gcmkONERROR(gckOS_AcquireMutex(Kernel->os,
Kernel->device->commitMutex,
gcvINFINITE
));
commitMutexAcquired = gcvTRUE;
}
/* Commit a command and context buffer. */
if (Interface->engine == gcvENGINE_BLT)
{
gctUINT64 *commandBuffers = gcmUINT64_TO_PTR(Interface->u.Commit.commandBuffer);
if (!gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_ASYNC_BLIT))
{
gcmkONERROR(gcvSTATUS_NOT_SUPPORTED);
}
gcmkONERROR(gckASYNC_COMMAND_Commit(
Kernel->asyncCommand,
gcmUINT64_TO_PTR(commandBuffers[0]),
gcmUINT64_TO_PTR(Interface->u.Commit.queue)
));
gcmkONERROR(gckEVENT_Commit(
Kernel->asyncEvent,
gcmUINT64_TO_PTR(Interface->u.Commit.queue),
gcvFALSE
));
}
else
{
gctUINT32 i;
if (Interface->u.Commit.count > 1 && Interface->engine == gcvENGINE_RENDER)
{
for (i = 0; i < Interface->u.Commit.count; i++)
{
gceHARDWARE_TYPE type = Interface->hardwareType;
gckKERNEL kernel = Device->map[type].kernels[i];
gcmkONERROR(gckOS_Broadcast(kernel->os,
kernel->hardware,
gcvBROADCAST_GPU_COMMIT));
}
}
status = gckCOMMAND_Commit(Kernel->command,
Interface->u.Commit.contexts[0] ?
gcmNAME_TO_PTR(Interface->u.Commit.contexts[0]) : gcvNULL,
gcmUINT64_TO_PTR(Interface->u.Commit.commandBuffers[0]),
gcmUINT64_TO_PTR(Interface->u.Commit.deltas[0]),
processID,
Interface->u.Commit.shared,
Interface->u.Commit.index,
&Interface->u.Commit.commitStamp,
&Interface->u.Commit.contextSwitched
);
if (status != gcvSTATUS_INTERRUPTED)
{
gcmkONERROR(status);
}
/* Force an event if powerManagement is on. */
status = gckEVENT_Commit(Kernel->eventObj,
gcmUINT64_TO_PTR(Interface->u.Commit.queue),
Kernel->hardware->options.powerManagement);
if (status != gcvSTATUS_INTERRUPTED)
{
gcmkONERROR(status);
}
if (Interface->u.Commit.count > 1 && Interface->engine == gcvENGINE_RENDER)
{
for (i = 1; i < Interface->u.Commit.count; i++)
{
gceHARDWARE_TYPE type = Interface->hardwareType;
gckKERNEL kernel = Device->map[type].kernels[i];
status = gckCOMMAND_Commit(kernel->command,
Interface->u.Commit.contexts[i] ?
gcmNAME_TO_PTR(Interface->u.Commit.contexts[i]) : gcvNULL,
Interface->u.Commit.commandBuffers[i] ?
gcmUINT64_TO_PTR(Interface->u.Commit.commandBuffers[i]) : gcmUINT64_TO_PTR(Interface->u.Commit.commandBuffers[0]),
gcmUINT64_TO_PTR(Interface->u.Commit.deltas[i]),
processID,
Interface->u.Commit.shared,
Interface->u.Commit.commandBuffers[i] ?
Interface->u.Commit.index : i,
&Interface->u.Commit.commitStamp,
&Interface->u.Commit.contextSwitched
);
if (status != gcvSTATUS_INTERRUPTED)
{
gcmkONERROR(status);
}
/* Force an event if powerManagement is on. */
status = gckEVENT_Commit(kernel->eventObj,
gcvNULL,
kernel->hardware->options.powerManagement);
if (status != gcvSTATUS_INTERRUPTED)
{
gcmkONERROR(status);
}
}
}
for (i = 0; i < Interface->u.Commit.count; i++)
{
gceHARDWARE_TYPE type = Interface->hardwareType;
gckKERNEL kernel = Device->map[type].kernels[i];
if ((kernel->hardware->options.gpuProfiler == gcvTRUE) &&
(kernel->profileEnable == gcvTRUE))
{
gcmkONERROR(gckCOMMAND_Stall(kernel->command, gcvTRUE));
if (kernel->command->currContext)
{
gcmkONERROR(gckHARDWARE_UpdateContextProfile(
kernel->hardware,
kernel->command->currContext));
}
}
}
}
if (!Interface->commitMutex)
{
gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->device->commitMutex));
commitMutexAcquired = gcvFALSE;
}
break;
case gcvHAL_STALL:
/* Stall the command queue. */
gcmkONERROR(gckCOMMAND_Stall(Kernel->command, gcvFALSE));
break;
case gcvHAL_MAP_USER_MEMORY:
gcmkONERROR(gcvSTATUS_NOT_SUPPORTED);
break;
case gcvHAL_UNMAP_USER_MEMORY:
gcmkONERROR(gcvSTATUS_NOT_SUPPORTED);
break;
#if !USE_NEW_LINUX_SIGNAL
case gcvHAL_USER_SIGNAL:
/* Dispatch depends on the user signal subcommands. */
switch(Interface->u.UserSignal.command)
{
case gcvUSER_SIGNAL_CREATE:
/* Create a signal used in the user space. */
gcmkONERROR(
gckOS_CreateUserSignal(Kernel->os,
Interface->u.UserSignal.manualReset,
&Interface->u.UserSignal.id));
gcmkVERIFY_OK(
gckKERNEL_AddProcessDB(Kernel,
processID, gcvDB_SIGNAL,
gcmINT2PTR(Interface->u.UserSignal.id),
gcvNULL,
0));
break;
case gcvUSER_SIGNAL_DESTROY:
gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
Kernel,
processID, gcvDB_SIGNAL,
gcmINT2PTR(Interface->u.UserSignal.id)));
/* Destroy the signal. */
gcmkONERROR(
gckOS_DestroyUserSignal(Kernel->os,
Interface->u.UserSignal.id));
break;
case gcvUSER_SIGNAL_SIGNAL:
/* Signal the signal. */
gcmkONERROR(
gckOS_SignalUserSignal(Kernel->os,
Interface->u.UserSignal.id,
Interface->u.UserSignal.state));
break;
case gcvUSER_SIGNAL_WAIT:
/* Wait on the signal. */
status = gckOS_WaitUserSignal(Kernel->os,
Interface->u.UserSignal.id,
Interface->u.UserSignal.wait);
break;
case gcvUSER_SIGNAL_MAP:
gcmkONERROR(
gckOS_MapSignal(Kernel->os,
(gctSIGNAL)(gctUINTPTR_T)Interface->u.UserSignal.id,
(gctHANDLE)(gctUINTPTR_T)processID,
&signal));
gcmkVERIFY_OK(
gckKERNEL_AddProcessDB(Kernel,
processID, gcvDB_SIGNAL,
gcmINT2PTR(Interface->u.UserSignal.id),
gcvNULL,
0));
break;
case gcvUSER_SIGNAL_UNMAP:
gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
Kernel,
processID, gcvDB_SIGNAL,
gcmINT2PTR(Interface->u.UserSignal.id)));
/* Destroy the signal. */
gcmkONERROR(
gckOS_DestroyUserSignal(Kernel->os,
Interface->u.UserSignal.id));
break;
default:
/* Invalid user signal command. */
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
break;
#endif
case gcvHAL_SET_POWER_MANAGEMENT_STATE:
/* Set the power management state. */
gcmkONERROR(
gckHARDWARE_SetPowerManagementState(
Kernel->hardware,
Interface->u.SetPowerManagement.state));
break;
case gcvHAL_QUERY_POWER_MANAGEMENT_STATE:
/* Chip is not idle. */
Interface->u.QueryPowerManagement.isIdle = gcvFALSE;
/* Query the power management state. */
gcmkONERROR(gckHARDWARE_QueryPowerManagementState(
Kernel->hardware,
&Interface->u.QueryPowerManagement.state));
/* Query the idle state. */
gcmkONERROR(
gckHARDWARE_QueryIdle(Kernel->hardware,
&Interface->u.QueryPowerManagement.isIdle));
break;
case gcvHAL_READ_REGISTER:
#if gcdREGISTER_ACCESS_FROM_USER
{
gceCHIPPOWERSTATE power;
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->hardware->powerMutex, gcvINFINITE));
powerMutexAcquired = gcvTRUE;
gcmkONERROR(gckHARDWARE_QueryPowerManagementState(Kernel->hardware,
&power));
if (power == gcvPOWER_ON)
{
/* Read a register. */
gcmkONERROR(gckOS_ReadRegisterEx(
Kernel->os,
Kernel->core,
Interface->u.ReadRegisterData.address,
&Interface->u.ReadRegisterData.data));
}
else
{
/* Chip is in power-state. */
Interface->u.ReadRegisterData.data = 0;
status = gcvSTATUS_CHIP_NOT_READY;
}
gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex));
powerMutexAcquired = gcvFALSE;
}
#else
/* No access from user land to read registers. */
Interface->u.ReadRegisterData.data = 0;
status = gcvSTATUS_NOT_SUPPORTED;
#endif
break;
case gcvHAL_WRITE_REGISTER:
#if gcdREGISTER_ACCESS_FROM_USER
{
gceCHIPPOWERSTATE power;
gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->hardware->powerMutex, gcvINFINITE));
powerMutexAcquired = gcvTRUE;
gcmkONERROR(gckHARDWARE_QueryPowerManagementState(Kernel->hardware,
&power));
if (power == gcvPOWER_ON)
{
/* Write a register. */
gcmkONERROR(
gckOS_WriteRegisterEx(Kernel->os,
Kernel->core,
Interface->u.WriteRegisterData.address,
Interface->u.WriteRegisterData.data));
}
else
{
/* Chip is in power-state. */
Interface->u.WriteRegisterData.data = 0;
status = gcvSTATUS_CHIP_NOT_READY;
}
gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex));
powerMutexAcquired = gcvFALSE;
}
#else
/* No access from user land to write registers. */
status = gcvSTATUS_NOT_SUPPORTED;
#endif
break;
case gcvHAL_READ_ALL_PROFILE_REGISTERS_PART1:
/* Read profile data according to the context. */
gcmkONERROR(
gckHARDWARE_QueryContextProfile(
Kernel->hardware,
Kernel->profileCleanRegister,
gcmNAME_TO_PTR(Interface->u.RegisterProfileData_part1.context),
&Interface->u.RegisterProfileData_part1.Counters,
gcvNULL));
break;
case gcvHAL_READ_ALL_PROFILE_REGISTERS_PART2:
/* Read profile data according to the context. */
gcmkONERROR(
gckHARDWARE_QueryContextProfile(
Kernel->hardware,
Kernel->profileCleanRegister,
gcmNAME_TO_PTR(Interface->u.RegisterProfileData_part2.context),
gcvNULL,
&Interface->u.RegisterProfileData_part2.Counters));
break;
case gcvHAL_GET_PROFILE_SETTING:
#if VIVANTE_PROFILER
/* Get profile setting */
Interface->u.GetProfileSetting.enable = Kernel->profileEnable;
#endif
status = gcvSTATUS_OK;
break;
case gcvHAL_SET_PROFILE_SETTING:
#if VIVANTE_PROFILER
/* Set profile setting */
if(Kernel->hardware->options.gpuProfiler)
{
Kernel->profileEnable = Interface->u.SetProfileSetting.enable;
if (Kernel->profileEnable)
{
gcmkONERROR(gckHARDWARE_InitProfiler(Kernel->hardware));
}
}
else
{
status = gcvSTATUS_NOT_SUPPORTED;
break;
}
#endif
status = gcvSTATUS_OK;
break;
case gcvHAL_READ_PROFILER_REGISTER_SETTING:
Kernel->profileCleanRegister = Interface->u.SetProfilerRegisterClear.bclear;
status = gcvSTATUS_OK;
break;
case gcvHAL_QUERY_KERNEL_SETTINGS:
/* Get kernel settings. */
gcmkONERROR(
gckKERNEL_QuerySettings(Kernel,
&Interface->u.QueryKernelSettings.settings));
break;
case gcvHAL_RESET:
/* Reset the hardware. */
gcmkONERROR(
gckHARDWARE_Reset(Kernel->hardware));
break;
case gcvHAL_DEBUG:
/* Set debug level and zones. */
if (Interface->u.Debug.set)
{
gckOS_SetDebugLevel(Interface->u.Debug.level);
gckOS_SetDebugZones(Interface->u.Debug.zones,
Interface->u.Debug.enable);
}
if (Interface->u.Debug.message[0] != '\0')
{
/* Print a message to the debugger. */
if (Interface->u.Debug.type == gcvMESSAGE_TEXT)
{
gckOS_DumpBuffer(Kernel->os,
Interface->u.Debug.message,
gcmSIZEOF(Interface->u.Debug.message),
gcvDUMP_BUFFER_FROM_USER,
gcvTRUE);
}
else
{
gckOS_DumpBuffer(Kernel->os,
Interface->u.Debug.message,
Interface->u.Debug.messageSize,
gcvDUMP_BUFFER_FROM_USER,
gcvTRUE);
}
}
status = gcvSTATUS_OK;
break;
case gcvHAL_DUMP_GPU_STATE:
{
gceCHIPPOWERSTATE power;
_DumpDriverConfigure(Kernel);
gcmkONERROR(gckHARDWARE_QueryPowerManagementState(
Kernel->hardware,
&power
));
if (power == gcvPOWER_ON)
{
Interface->u.ReadRegisterData.data = 1;
_DumpState(Kernel);
}
else
{
Interface->u.ReadRegisterData.data = 0;
status = gcvSTATUS_CHIP_NOT_READY;
gcmkPRINT("[galcore]: Can't dump state if GPU isn't POWER ON.");
}
}
break;
case gcvHAL_DUMP_EVENT:
break;
case gcvHAL_CACHE:
logical = gcmUINT64_TO_PTR(Interface->u.Cache.logical);
bytes = (gctSIZE_T) Interface->u.Cache.bytes;
gcmkONERROR(gckKERNEL_CacheOperation(Kernel,
processID,
Interface->u.Cache.node,
Interface->u.Cache.operation,
logical,
bytes));
break;
case gcvHAL_TIMESTAMP:
/* Check for invalid timer. */
if ((Interface->u.TimeStamp.timer >= gcmCOUNTOF(Kernel->timers))
|| (Interface->u.TimeStamp.request != 2))
{
Interface->u.TimeStamp.timeDelta = 0;
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
/* Return timer results and reset timer. */
{
gcsTIMER_PTR timer = &(Kernel->timers[Interface->u.TimeStamp.timer]);
gctUINT64 timeDelta = 0;
if (timer->stopTime < timer->startTime )
{
Interface->u.TimeStamp.timeDelta = 0;
gcmkONERROR(gcvSTATUS_TIMER_OVERFLOW);
}
timeDelta = timer->stopTime - timer->startTime;
/* Check truncation overflow. */
Interface->u.TimeStamp.timeDelta = (gctINT32) timeDelta;
/*bit0~bit30 is available*/
if (timeDelta>>31)
{
Interface->u.TimeStamp.timeDelta = 0;
gcmkONERROR(gcvSTATUS_TIMER_OVERFLOW);
}
status = gcvSTATUS_OK;
}
break;
case gcvHAL_DATABASE:
gcmkONERROR(gckKERNEL_QueryDatabase(Kernel, processID, Interface));
break;
case gcvHAL_VERSION:
Interface->u.Version.major = gcvVERSION_MAJOR;
Interface->u.Version.minor = gcvVERSION_MINOR;
Interface->u.Version.patch = gcvVERSION_PATCH;
Interface->u.Version.build = gcvVERSION_BUILD;
#if gcmIS_DEBUG(gcdDEBUG_TRACE)
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL,
"KERNEL version %d.%d.%d build %u",
gcvVERSION_MAJOR, gcvVERSION_MINOR,
gcvVERSION_PATCH, gcvVERSION_BUILD);
#endif
break;
case gcvHAL_CHIP_INFO:
/* Only if not support multi-core */
Interface->u.ChipInfo.count = 1;
Interface->u.ChipInfo.types[0] = Kernel->hardware->type;
break;
#if (gcdENABLE_3D || gcdENABLE_2D)
case gcvHAL_ATTACH:
/* Attach user process. */
gcmkONERROR(
gckCOMMAND_Attach(Kernel->command,
&context,
&bytes,
&Interface->u.Attach.numStates,
processID));
Interface->u.Attach.maxState = bytes;
Interface->u.Attach.context = gcmPTR_TO_NAME(context);
if (Interface->u.Attach.map == gcvTRUE)
{
gcmkVERIFY_OK(
gckCONTEXT_MapBuffer(context,
Interface->u.Attach.physicals,
Interface->u.Attach.logicals,
&Interface->u.Attach.bytes));
}
gcmkVERIFY_OK(
gckKERNEL_AddProcessDB(Kernel,
processID, gcvDB_CONTEXT,
gcmINT2PTR(Interface->u.Attach.context),
gcvNULL,
0));
break;
#endif
case gcvHAL_DETACH:
gcmkVERIFY_OK(
gckKERNEL_RemoveProcessDB(Kernel,
processID, gcvDB_CONTEXT,
gcmINT2PTR(Interface->u.Detach.context)));
/* Detach user process. */
gcmkONERROR(
gckCOMMAND_Detach(Kernel->command,
gcmNAME_TO_PTR(Interface->u.Detach.context)));
gcmRELEASE_NAME(Interface->u.Detach.context);
gcmkONERROR(gckOS_AcquireMutex(Kernel->os,
Kernel->device->commitMutex,
gcvINFINITE
));
commitMutexAcquired = gcvTRUE;
gcmkONERROR(gckEVENT_Submit(
Kernel->eventObj,
gcvTRUE,
gcvFALSE
));
gcmkONERROR(gckOS_ReleaseMutex(Kernel->os,
Kernel->device->commitMutex
));
commitMutexAcquired = gcvFALSE;
break;
case gcvHAL_GET_FRAME_INFO:
gcmkONERROR(gckHARDWARE_GetFrameInfo(
Kernel->hardware,
gcmUINT64_TO_PTR(Interface->u.GetFrameInfo.frameInfo)));
break;
case gcvHAL_DUMP_GPU_PROFILE:
gcmkONERROR(gckHARDWARE_DumpGpuProfile(Kernel->hardware));
break;
case gcvHAL_SET_FSCALE_VALUE:
#if gcdENABLE_FSCALE_VAL_ADJUST
status = gckHARDWARE_SetFscaleValue(Kernel->hardware,
Interface->u.SetFscaleValue.value);
#else
status = gcvSTATUS_NOT_SUPPORTED;
#endif
break;
case gcvHAL_GET_FSCALE_VALUE:
#if gcdENABLE_FSCALE_VAL_ADJUST
status = gckHARDWARE_GetFscaleValue(Kernel->hardware,
&Interface->u.GetFscaleValue.value,
&Interface->u.GetFscaleValue.minValue,
&Interface->u.GetFscaleValue.maxValue);
#else
status = gcvSTATUS_NOT_SUPPORTED;
#endif
break;
case gcvHAL_EXPORT_VIDEO_MEMORY:
/* Unlock video memory. */
gcmkONERROR(gckVIDMEM_NODE_Export(Kernel,
Interface->u.ExportVideoMemory.node,
Interface->u.ExportVideoMemory.flags,
gcvNULL,
&Interface->u.ExportVideoMemory.fd));
break;
case gcvHAL_NAME_VIDEO_MEMORY:
gcmkONERROR(gckVIDMEM_NODE_Name(Kernel,
Interface->u.NameVideoMemory.handle,
&Interface->u.NameVideoMemory.name));
break;
case gcvHAL_IMPORT_VIDEO_MEMORY:
gcmkONERROR(gckVIDMEM_NODE_Import(Kernel,
Interface->u.ImportVideoMemory.name,
&Interface->u.ImportVideoMemory.handle));
gcmkONERROR(
gckKERNEL_AddProcessDB(Kernel,
processID, gcvDB_VIDEO_MEMORY,
gcmINT2PTR(Interface->u.ImportVideoMemory.handle),
gcvNULL,
0));
break;
case gcvHAL_SET_VIDEO_MEMORY_METADATA:
gcmkONERROR(gckKERNEL_SetVidMemMetadata(Kernel, processID, Interface));
break;
case gcvHAL_GET_VIDEO_MEMORY_FD:
gcmkONERROR(gckVIDMEM_NODE_GetFd(
Kernel,
Interface->u.GetVideoMemoryFd.handle,
&Interface->u.GetVideoMemoryFd.fd
));
/* No need to add it to processDB because OS will release all fds when
** process quits.
*/
break;
case gcvHAL_QUERY_RESET_TIME_STAMP:
Interface->u.QueryResetTimeStamp.timeStamp = Kernel->resetTimeStamp;
Interface->u.QueryResetTimeStamp.contextID = Kernel->hardware->contextID;
break;
case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER:
buffer = (gckVIRTUAL_COMMAND_BUFFER_PTR)gcmNAME_TO_PTR(Interface->u.FreeVirtualCommandBuffer.physical);
gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
Kernel,
processID,
gcvDB_COMMAND_BUFFER,
gcmUINT64_TO_PTR(Interface->u.FreeVirtualCommandBuffer.logical)));
gcmkONERROR(gckOS_DestroyUserVirtualMapping(
Kernel->os,
buffer->virtualBuffer.physical,
(gctSIZE_T)Interface->u.FreeVirtualCommandBuffer.bytes,
gcmUINT64_TO_PTR(Interface->u.FreeVirtualCommandBuffer.logical)));
gcmkONERROR(gckKERNEL_DestroyVirtualCommandBuffer(
Kernel,
(gctSIZE_T)Interface->u.FreeVirtualCommandBuffer.bytes,
(gctPHYS_ADDR)buffer,
gcmUINT64_TO_PTR(Interface->u.FreeVirtualCommandBuffer.logical)));
gcmRELEASE_NAME(Interface->u.FreeVirtualCommandBuffer.physical);
break;
#if gcdLINUX_SYNC_FILE
case gcvHAL_CREATE_NATIVE_FENCE:
{
gctINT fenceFD;
gctSIGNAL signal =
gcmUINT64_TO_PTR(Interface->u.CreateNativeFence.signal);
gcmkONERROR(
gckOS_CreateNativeFence(Kernel->os,
Kernel->timeline,
signal,
&fenceFD));
Interface->u.CreateNativeFence.fenceFD = fenceFD;
}
break;
case gcvHAL_WAIT_NATIVE_FENCE:
{
gctINT fenceFD;
gctUINT32 timeout;
fenceFD = Interface->u.WaitNativeFence.fenceFD;
timeout = Interface->u.WaitNativeFence.timeout;
gcmkONERROR(
gckOS_WaitNativeFence(Kernel->os,
Kernel->timeline,
fenceFD,
timeout));
}
break;
#endif
case gcvHAL_SHBUF:
{
gctSHBUF shBuf;
gctPOINTER uData;
gctUINT32 bytes;
switch (Interface->u.ShBuf.command)
{
case gcvSHBUF_CREATE:
bytes = Interface->u.ShBuf.bytes;
/* Create. */
gcmkONERROR(gckKERNEL_CreateShBuffer(Kernel, bytes, &shBuf));
Interface->u.ShBuf.id = gcmPTR_TO_UINT64(shBuf);
gcmkVERIFY_OK(
gckKERNEL_AddProcessDB(Kernel,
processID,
gcvDB_SHBUF,
shBuf,
gcvNULL,
0));
break;
case gcvSHBUF_DESTROY:
shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id);
/* Check db first to avoid illegal destroy in the process. */
gcmkONERROR(
gckKERNEL_RemoveProcessDB(Kernel,
processID,
gcvDB_SHBUF,
shBuf));
gcmkONERROR(gckKERNEL_DestroyShBuffer(Kernel, shBuf));
break;
case gcvSHBUF_MAP:
shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id);
/* Map for current process access. */
gcmkONERROR(gckKERNEL_MapShBuffer(Kernel, shBuf));
gcmkVERIFY_OK(
gckKERNEL_AddProcessDB(Kernel,
processID,
gcvDB_SHBUF,
shBuf,
gcvNULL,
0));
break;
case gcvSHBUF_WRITE:
shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id);
uData = gcmUINT64_TO_PTR(Interface->u.ShBuf.data);
bytes = Interface->u.ShBuf.bytes;
/* Write. */
gcmkONERROR(
gckKERNEL_WriteShBuffer(Kernel, shBuf, uData, bytes));
break;
case gcvSHBUF_READ:
shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id);
uData = gcmUINT64_TO_PTR(Interface->u.ShBuf.data);
bytes = Interface->u.ShBuf.bytes;
/* Read. */
gcmkONERROR(
gckKERNEL_ReadShBuffer(Kernel,
shBuf,
uData,
bytes,
&bytes));
/* Return copied size. */
Interface->u.ShBuf.bytes = bytes;
break;
default:
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
break;
}
}
break;
#ifdef __linux__
case gcvHAL_GET_GRAPHIC_BUFFER_FD:
gcmkONERROR(_GetGraphicBufferFd(
Kernel,
processID,
Interface->u.GetGraphicBufferFd.node,
Interface->u.GetGraphicBufferFd.shBuf,
Interface->u.GetGraphicBufferFd.signal,
&Interface->u.GetGraphicBufferFd.fd
));
break;
#endif
case gcvHAL_CONFIG_POWER_MANAGEMENT:
gcmkONERROR(gckKERNEL_ConfigPowerManagement(Kernel, Interface));
break;
case gcvHAL_WRAP_USER_MEMORY:
gcmkONERROR(gckVIDMEM_NODE_WrapUserMemory(Kernel,
&Interface->u.WrapUserMemory.desc,
&Interface->u.WrapUserMemory.node,
&Interface->u.WrapUserMemory.bytes));
gcmkONERROR(
gckKERNEL_AddProcessDB(Kernel,
processID,
gcvDB_VIDEO_MEMORY,
gcmINT2PTR(Interface->u.WrapUserMemory.node),
gcvNULL,
0));
break;
case gcvHAL_WAIT_FENCE:
gcmkONERROR(gckKERNEL_WaitFence(
Kernel,
Interface->u.WaitFence.handle,
Interface->u.WaitFence.timeOut
));
break;
case gcvHAL_DEVICE_MUTEX:
if (Interface->u.DeviceMutex.isMutexLocked)
{
gcmkONERROR(gckOS_AcquireMutex(Kernel->os,
Kernel->device->commitMutex,
gcvINFINITE
));
}
else
{
gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->device->commitMutex));
}
break;
#if gcdDEC_ENABLE_AHB
case gcvHAL_DEC300_READ:
gcmkONERROR(viv_dec300_read(
Interface->u.DEC300Read.enable,
Interface->u.DEC300Read.readId,
Interface->u.DEC300Read.format,
Interface->u.DEC300Read.strides,
Interface->u.DEC300Read.is3D,
Interface->u.DEC300Read.isMSAA,
Interface->u.DEC300Read.clearValue,
Interface->u.DEC300Read.isTPC,
Interface->u.DEC300Read.isTPCCompressed,
Interface->u.DEC300Read.surfAddrs,
Interface->u.DEC300Read.tileAddrs
));
break;
case gcvHAL_DEC300_WRITE:
gcmkONERROR(viv_dec300_write(
Interface->u.DEC300Write.enable,
Interface->u.DEC300Write.readId,
Interface->u.DEC300Write.writeId,
Interface->u.DEC300Write.format,
Interface->u.DEC300Write.surfAddr,
Interface->u.DEC300Write.tileAddr
));
break;
case gcvHAL_DEC300_FLUSH:
gcmkONERROR(viv_dec300_flush(0));
break;
case gcvHAL_DEC300_FLUSH_WAIT:
gcmkONERROR(viv_dec300_flush_done(&Interface->u.DEC300FlushWait.done));
break;
#endif
case gcvHAL_QUERY_CHIP_OPTION:
/* Query chip options. */
gcmkONERROR(
gckHARDWARE_QueryChipOptions(
Kernel->hardware,
&Interface->u.QueryChipOptions));
break;
default:
/* Invalid command. */
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
OnError:
/* Save status. */
Interface->status = status;
#if QNX_SINGLE_THREADED_DEBUGGING
gckOS_ReleaseMutex(Kernel->os, Kernel->debugMutex);
#endif
if (powerMutexAcquired == gcvTRUE)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex));
}
if (commitMutexAcquired == gcvTRUE)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->device->commitMutex));
}
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
** gckKERNEL_AttachProcess
**
** Attach or detach a process.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctBOOL Attach
** gcvTRUE if a new process gets attached or gcFALSE when a process
** gets detatched.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckKERNEL_AttachProcess(
IN gckKERNEL Kernel,
IN gctBOOL Attach
)
{
gceSTATUS status;
gctUINT32 processID;
gcmkHEADER_ARG("Kernel=0x%x Attach=%d", Kernel, Attach);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
/* Get current process ID. */
gcmkONERROR(gckOS_GetProcessID(&processID));
gcmkONERROR(gckKERNEL_AttachProcessEx(Kernel, Attach, processID));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
** gckKERNEL_AttachProcessEx
**
** Attach or detach a process with the given PID. Can be paired with gckKERNEL_AttachProcess
** provided the programmer is aware of the consequences.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctBOOL Attach
** gcvTRUE if a new process gets attached or gcFALSE when a process
** gets detatched.
**
** gctUINT32 PID
** PID of the process to attach or detach.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckKERNEL_AttachProcessEx(
IN gckKERNEL Kernel,
IN gctBOOL Attach,
IN gctUINT32 PID
)
{
gceSTATUS status;
gctINT32 old;
gcmkHEADER_ARG("Kernel=0x%x Attach=%d PID=%d", Kernel, Attach, PID);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
if (Attach)
{
/* Increment the number of clients attached. */
gcmkONERROR(
gckOS_AtomIncrement(Kernel->os, Kernel->atomClients, &old));
if (old == 0)
{
#if gcdENABLE_VG
if (Kernel->vg == gcvNULL)
#endif
{
gcmkONERROR(gckOS_Broadcast(Kernel->os,
Kernel->hardware,
gcvBROADCAST_FIRST_PROCESS));
}
}
if (Kernel->dbCreated)
{
/* Create the process database. */
gcmkONERROR(gckKERNEL_CreateProcessDB(Kernel, PID));
}
#if gcdPROCESS_ADDRESS_SPACE
/* Map kernel command buffer in the process's own MMU. */
gcmkONERROR(_MapCommandBuffer(Kernel));
#endif
}
else
{
if (Kernel->dbCreated)
{
/* Clean up the process database. */
gcmkONERROR(gckKERNEL_DestroyProcessDB(Kernel, PID));
/* Save the last know process ID. */
Kernel->db->lastProcessID = PID;
}
#if gcdENABLE_VG
if (Kernel->vg == gcvNULL)
#endif
{
status = gckEVENT_Submit(Kernel->eventObj, gcvTRUE, gcvFALSE);
if (status == gcvSTATUS_INTERRUPTED && Kernel->eventObj->submitTimer)
{
gcmkONERROR(gckOS_StartTimer(Kernel->os,
Kernel->eventObj->submitTimer,
1));
}
else
{
gcmkONERROR(status);
}
}
/* Decrement the number of clients attached. */
gcmkONERROR(
gckOS_AtomDecrement(Kernel->os, Kernel->atomClients, &old));
if (old == 1)
{
#if gcdENABLE_VG
if (Kernel->vg == gcvNULL)
#endif
{
/* Last client detached, switch to SUSPEND power state. */
gcmkONERROR(gckOS_Broadcast(Kernel->os,
Kernel->hardware,
gcvBROADCAST_LAST_PROCESS));
}
/* Flush the debug cache. */
gcmkDEBUGFLUSH(~0U);
}
}
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
#if gcdSECURE_USER
gceSTATUS
gckKERNEL_MapLogicalToPhysical(
IN gckKERNEL Kernel,
IN gcskSECURE_CACHE_PTR Cache,
IN OUT gctPOINTER * Data
)
{
gceSTATUS status;
static gctBOOL baseAddressValid = gcvFALSE;
static gctUINT32 baseAddress;
gctBOOL needBase;
gcskLOGICAL_CACHE_PTR slot;
gcmkHEADER_ARG("Kernel=0x%x Cache=0x%x *Data=0x%x",
Kernel, Cache, gcmOPT_POINTER(Data));
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
if (!baseAddressValid)
{
/* Get base address. */
gcmkONERROR(gckHARDWARE_GetBaseAddress(Kernel->hardware, &baseAddress));
baseAddressValid = gcvTRUE;
}
/* Does this state load need a base address? */
gcmkONERROR(gckHARDWARE_NeedBaseAddress(Kernel->hardware,
((gctUINT32_PTR) Data)[-1],
&needBase));
#if gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_LRU
{
gcskLOGICAL_CACHE_PTR next;
gctINT i;
/* Walk all used cache slots. */
for (i = 1, slot = Cache->cache[0].next, next = gcvNULL;
(i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
++i, slot = slot->next
)
{
if (slot->logical == *Data)
{
/* Bail out. */
next = slot;
break;
}
}
/* See if we had a miss. */
if (next == gcvNULL)
{
/* Use the tail of the cache. */
slot = Cache->cache[0].prev;
/* Initialize the cache line. */
slot->logical = *Data;
/* Map the logical address to a DMA address. */
gcmkONERROR(
gckOS_GetPhysicalAddress(Kernel->os, *Data, &slot->dma));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Kernel->os, slot->dma, &slot->dma));
}
/* Move slot to head of list. */
if (slot != Cache->cache[0].next)
{
/* Unlink. */
slot->prev->next = slot->next;
slot->next->prev = slot->prev;
/* Move to head of chain. */
slot->prev = &Cache->cache[0];
slot->next = Cache->cache[0].next;
slot->prev->next = slot;
slot->next->prev = slot;
}
}
#elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_LINEAR
{
gctINT i;
gcskLOGICAL_CACHE_PTR next = gcvNULL;
gcskLOGICAL_CACHE_PTR oldestSlot = gcvNULL;
slot = gcvNULL;
if (Cache->cacheIndex != gcvNULL)
{
/* Walk the cache forwards. */
for (i = 1, slot = Cache->cacheIndex;
(i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
++i, slot = slot->next)
{
if (slot->logical == *Data)
{
/* Bail out. */
next = slot;
break;
}
/* Determine age of this slot. */
if ((oldestSlot == gcvNULL)
|| (oldestSlot->stamp > slot->stamp)
)
{
oldestSlot = slot;
}
}
if (next == gcvNULL)
{
/* Walk the cache backwards. */
for (slot = Cache->cacheIndex->prev;
(i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
++i, slot = slot->prev)
{
if (slot->logical == *Data)
{
/* Bail out. */
next = slot;
break;
}
/* Determine age of this slot. */
if ((oldestSlot == gcvNULL)
|| (oldestSlot->stamp > slot->stamp)
)
{
oldestSlot = slot;
}
}
}
}
/* See if we had a miss. */
if (next == gcvNULL)
{
if (Cache->cacheFree != 0)
{
slot = &Cache->cache[Cache->cacheFree];
gcmkASSERT(slot->logical == gcvNULL);
++ Cache->cacheFree;
if (Cache->cacheFree >= gcmCOUNTOF(Cache->cache))
{
Cache->cacheFree = 0;
}
}
else
{
/* Use the oldest cache slot. */
gcmkASSERT(oldestSlot != gcvNULL);
slot = oldestSlot;
/* Unlink from the chain. */
slot->prev->next = slot->next;
slot->next->prev = slot->prev;
/* Append to the end. */
slot->prev = Cache->cache[0].prev;
slot->next = &Cache->cache[0];
slot->prev->next = slot;
slot->next->prev = slot;
}
/* Initialize the cache line. */
slot->logical = *Data;
/* Map the logical address to a DMA address. */
gcmkONERROR(
gckOS_GetPhysicalAddress(Kernel->os, *Data, &slot->dma));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Kernel->os, slot->dma, &slot->dma));
}
/* Save time stamp. */
slot->stamp = ++ Cache->cacheStamp;
/* Save current slot for next lookup. */
Cache->cacheIndex = slot;
}
#elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_HASH
{
gctINT i;
gctUINT32 data = gcmPTR2INT32(*Data);
gctUINT32 key, index;
gcskLOGICAL_CACHE_PTR hash;
/* Generate a hash key. */
key = (data >> 24) + (data >> 16) + (data >> 8) + data;
index = key % gcmCOUNTOF(Cache->hash);
/* Get the hash entry. */
hash = &Cache->hash[index];
for (slot = hash->nextHash, i = 0;
(slot != gcvNULL) && (i < gcdSECURE_CACHE_SLOTS);
slot = slot->nextHash, ++i
)
{
if (slot->logical == (*Data))
{
break;
}
}
if (slot == gcvNULL)
{
/* Grab from the tail of the cache. */
slot = Cache->cache[0].prev;
/* Unlink slot from any hash table it is part of. */
if (slot->prevHash != gcvNULL)
{
slot->prevHash->nextHash = slot->nextHash;
}
if (slot->nextHash != gcvNULL)
{
slot->nextHash->prevHash = slot->prevHash;
}
/* Initialize the cache line. */
slot->logical = *Data;
/* Map the logical address to a DMA address. */
gcmkONERROR(
gckOS_GetPhysicalAddress(Kernel->os, *Data, &slot->dma));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Kernel->os, slot->dma, &slot->dma));
if (hash->nextHash != gcvNULL)
{
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL,
"Hash Collision: logical=0x%x key=0x%08x",
*Data, key);
}
/* Insert the slot at the head of the hash list. */
slot->nextHash = hash->nextHash;
if (slot->nextHash != gcvNULL)
{
slot->nextHash->prevHash = slot;
}
slot->prevHash = hash;
hash->nextHash = slot;
}
/* Move slot to head of list. */
if (slot != Cache->cache[0].next)
{
/* Unlink. */
slot->prev->next = slot->next;
slot->next->prev = slot->prev;
/* Move to head of chain. */
slot->prev = &Cache->cache[0];
slot->next = Cache->cache[0].next;
slot->prev->next = slot;
slot->next->prev = slot;
}
}
#elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_TABLE
{
gctUINT32 index = (gcmPTR2INT32(*Data) % gcdSECURE_CACHE_SLOTS) + 1;
/* Get cache slot. */
slot = &Cache->cache[index];
/* Check for cache miss. */
if (slot->logical != *Data)
{
/* Initialize the cache line. */
slot->logical = *Data;
/* Map the logical address to a DMA address. */
gcmkONERROR(
gckOS_GetPhysicalAddress(Kernel->os, *Data, &slot->dma));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Kernel->os, slot->dma, &slot->dma));
}
}
#endif
/* Return DMA address. */
*Data = gcmINT2PTR(slot->dma + (needBase ? baseAddress : 0));
/* Success. */
gcmkFOOTER_ARG("*Data=0x%08x", *Data);
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckKERNEL_FlushTranslationCache(
IN gckKERNEL Kernel,
IN gcskSECURE_CACHE_PTR Cache,
IN gctPOINTER Logical,
IN gctSIZE_T Bytes
)
{
gctINT i;
gcskLOGICAL_CACHE_PTR slot;
gctUINT8_PTR ptr;
gcmkHEADER_ARG("Kernel=0x%x Cache=0x%x Logical=0x%x Bytes=%lu",
Kernel, Cache, Logical, Bytes);
/* Do we need to flush the entire cache? */
if (Logical == gcvNULL)
{
/* Clear all cache slots. */
for (i = 1; i <= gcdSECURE_CACHE_SLOTS; ++i)
{
Cache->cache[i].logical = gcvNULL;
#if gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_HASH
Cache->cache[i].nextHash = gcvNULL;
Cache->cache[i].prevHash = gcvNULL;
#endif
}
#if gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_HASH
/* Zero the hash table. */
for (i = 0; i < gcmCOUNTOF(Cache->hash); ++i)
{
Cache->hash[i].nextHash = gcvNULL;
}
#endif
/* Reset the cache functionality. */
Cache->cacheIndex = gcvNULL;
Cache->cacheFree = 1;
Cache->cacheStamp = 0;
}
else
{
gctUINT8_PTR low = (gctUINT8_PTR) Logical;
gctUINT8_PTR high = low + Bytes;
#if gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_LRU
gcskLOGICAL_CACHE_PTR next;
/* Walk all used cache slots. */
for (i = 1, slot = Cache->cache[0].next;
(i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
++i, slot = next
)
{
/* Save pointer to next slot. */
next = slot->next;
/* Test if this slot falls within the range to flush. */
ptr = (gctUINT8_PTR) slot->logical;
if ((ptr >= low) && (ptr < high))
{
/* Unlink slot. */
slot->prev->next = slot->next;
slot->next->prev = slot->prev;
/* Append slot to tail of cache. */
slot->prev = Cache->cache[0].prev;
slot->next = &Cache->cache[0];
slot->prev->next = slot;
slot->next->prev = slot;
/* Mark slot as empty. */
slot->logical = gcvNULL;
}
}
#elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_LINEAR
gcskLOGICAL_CACHE_PTR next;
for (i = 1, slot = Cache->cache[0].next;
(i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
++i, slot = next)
{
/* Save pointer to next slot. */
next = slot->next;
/* Test if this slot falls within the range to flush. */
ptr = (gctUINT8_PTR) slot->logical;
if ((ptr >= low) && (ptr < high))
{
/* Test if this slot is the current slot. */
if (slot == Cache->cacheIndex)
{
/* Move to next or previous slot. */
Cache->cacheIndex = (slot->next->logical != gcvNULL)
? slot->next
: (slot->prev->logical != gcvNULL)
? slot->prev
: gcvNULL;
}
/* Unlink slot from cache. */
slot->prev->next = slot->next;
slot->next->prev = slot->prev;
/* Insert slot to head of cache. */
slot->prev = &Cache->cache[0];
slot->next = Cache->cache[0].next;
slot->prev->next = slot;
slot->next->prev = slot;
/* Mark slot as empty. */
slot->logical = gcvNULL;
slot->stamp = 0;
}
}
#elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_HASH
gctINT j;
gcskLOGICAL_CACHE_PTR hash, next;
/* Walk all hash tables. */
for (i = 0, hash = Cache->hash;
i < gcmCOUNTOF(Cache->hash);
++i, ++hash)
{
/* Walk all slots in the hash. */
for (j = 0, slot = hash->nextHash;
(j < gcdSECURE_CACHE_SLOTS) && (slot != gcvNULL);
++j, slot = next)
{
/* Save pointer to next slot. */
next = slot->next;
/* Test if this slot falls within the range to flush. */
ptr = (gctUINT8_PTR) slot->logical;
if ((ptr >= low) && (ptr < high))
{
/* Unlink slot from hash table. */
if (slot->prevHash == hash)
{
hash->nextHash = slot->nextHash;
}
else
{
slot->prevHash->nextHash = slot->nextHash;
}
if (slot->nextHash != gcvNULL)
{
slot->nextHash->prevHash = slot->prevHash;
}
/* Unlink slot from cache. */
slot->prev->next = slot->next;
slot->next->prev = slot->prev;
/* Append slot to tail of cache. */
slot->prev = Cache->cache[0].prev;
slot->next = &Cache->cache[0];
slot->prev->next = slot;
slot->next->prev = slot;
/* Mark slot as empty. */
slot->logical = gcvNULL;
slot->prevHash = gcvNULL;
slot->nextHash = gcvNULL;
}
}
}
#elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_TABLE
gctUINT32 index;
/* Loop while inside the range. */
for (i = 1; (low < high) && (i <= gcdSECURE_CACHE_SLOTS); ++i)
{
/* Get index into cache for this range. */
index = (gcmPTR2INT32(low) % gcdSECURE_CACHE_SLOTS) + 1;
slot = &Cache->cache[index];
/* Test if this slot falls within the range to flush. */
ptr = (gctUINT8_PTR) slot->logical;
if ((ptr >= low) && (ptr < high))
{
/* Remove entry from cache. */
slot->logical = gcvNULL;
}
/* Next block. */
low += gcdSECURE_CACHE_SLOTS;
}
#endif
}
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
#endif
/*******************************************************************************
**
** gckKERNEL_Recovery
**
** Try to recover the GPU from a fatal error.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckKERNEL_Recovery(
IN gckKERNEL Kernel
)
{
gceSTATUS status;
gckEVENT eventObj;
gckHARDWARE hardware;
#if gcdSECURE_USER
gctUINT32 processID;
gcskSECURE_CACHE_PTR cache;
#endif
gctUINT32 mask = 0;
gctUINT32 i = 0, count = 0;
#if gcdINTERRUPT_STATISTIC
gctINT32 oldValue;
#endif
gcmkHEADER_ARG("Kernel=0x%x", Kernel);
/* Validate the arguemnts. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
/* Grab gckEVENT object. */
eventObj = Kernel->eventObj;
gcmkVERIFY_OBJECT(eventObj, gcvOBJ_EVENT);
/* Grab gckHARDWARE object. */
hardware = Kernel->hardware;
gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
#if gcdSECURE_USER
/* Flush the secure mapping cache. */
gcmkONERROR(gckOS_GetProcessID(&processID));
gcmkONERROR(gckKERNEL_GetProcessDBCache(Kernel, processID, &cache));
gcmkONERROR(gckKERNEL_FlushTranslationCache(Kernel, cache, gcvNULL, 0));
#endif
if (Kernel->stuckDump == gcvSTUCK_DUMP_NONE)
{
gcmkPRINT("[galcore]: GPU[%d] hang, automatic recovery.", Kernel->core);
}
else
{
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, Kernel->device->stuckDumpMutex, gcvINFINITE));
_DumpDriverConfigure(Kernel);
_DumpState(Kernel);
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->device->stuckDumpMutex));
}
if (Kernel->recovery == gcvFALSE)
{
gcmkPRINT("[galcore]: Stop driver to keep scene.");
/* Stop monitor timer. */
Kernel->monitorTimerStop = gcvTRUE;
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/* Issuing a soft reset for the GPU. */
gcmkONERROR(gckHARDWARE_Reset(hardware));
mask = Kernel->restoreMask;
for (i = 0; i < 32; i++)
{
if (mask & (1 << i))
{
count++;
}
}
/* Handle all outstanding events now. */
gcmkONERROR(gckOS_AtomSet(Kernel->os, eventObj->pending, mask));
#if gcdINTERRUPT_STATISTIC
while (count--)
{
gcmkONERROR(gckOS_AtomDecrement(
Kernel->os,
eventObj->interruptCount,
&oldValue
));
}
gckOS_AtomClearMask(Kernel->hardware->pendingEvent, mask);
#endif
gcmkONERROR(gckEVENT_Notify(eventObj, 1));
gcmkVERIFY_OK(gckOS_GetTime(&Kernel->resetTimeStamp));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_OpenUserData
**
** Get access to the user data.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctBOOL NeedCopy
** The flag indicating whether or not the data should be copied.
**
** gctPOINTER StaticStorage
** Pointer to the kernel storage where the data is to be copied if
** NeedCopy is gcvTRUE.
**
** gctPOINTER UserPointer
** User pointer to the data.
**
** gctSIZE_T Size
** Size of the data.
**
** OUTPUT:
**
** gctPOINTER * KernelPointer
** Pointer to the kernel pointer that will be pointing to the data.
*/
gceSTATUS
gckKERNEL_OpenUserData(
IN gckKERNEL Kernel,
IN gctBOOL NeedCopy,
IN gctPOINTER StaticStorage,
IN gctPOINTER UserPointer,
IN gctSIZE_T Size,
OUT gctPOINTER * KernelPointer
)
{
gceSTATUS status;
gcmkHEADER_ARG(
"Kernel=0x%08X NeedCopy=%d StaticStorage=0x%08X "
"UserPointer=0x%08X Size=%lu KernelPointer=0x%08X",
Kernel, NeedCopy, StaticStorage, UserPointer, Size, KernelPointer
);
/* Validate the arguemnts. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(!NeedCopy || (StaticStorage != gcvNULL));
gcmkVERIFY_ARGUMENT(UserPointer != gcvNULL);
gcmkVERIFY_ARGUMENT(KernelPointer != gcvNULL);
gcmkVERIFY_ARGUMENT(Size > 0);
if (NeedCopy)
{
/* Copy the user data to the static storage. */
gcmkONERROR(gckOS_CopyFromUserData(
Kernel->os, StaticStorage, UserPointer, Size
));
/* Set the kernel pointer. */
* KernelPointer = StaticStorage;
}
else
{
gctPOINTER pointer = gcvNULL;
/* Map the user pointer. */
gcmkONERROR(gckOS_MapUserPointer(
Kernel->os, UserPointer, Size, &pointer
));
/* Set the kernel pointer. */
* KernelPointer = pointer;
}
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_CloseUserData
**
** Release resources associated with the user data connection opened by
** gckKERNEL_OpenUserData.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctBOOL NeedCopy
** The flag indicating whether or not the data should be copied.
**
** gctBOOL FlushData
** If gcvTRUE, the data is written back to the user.
**
** gctPOINTER UserPointer
** User pointer to the data.
**
** gctSIZE_T Size
** Size of the data.
**
** OUTPUT:
**
** gctPOINTER * KernelPointer
** Kernel pointer to the data.
*/
gceSTATUS
gckKERNEL_CloseUserData(
IN gckKERNEL Kernel,
IN gctBOOL NeedCopy,
IN gctBOOL FlushData,
IN gctPOINTER UserPointer,
IN gctSIZE_T Size,
OUT gctPOINTER * KernelPointer
)
{
gceSTATUS status = gcvSTATUS_OK;
gctPOINTER pointer;
gcmkHEADER_ARG(
"Kernel=0x%08X NeedCopy=%d FlushData=%d "
"UserPointer=0x%08X Size=%lu KernelPointer=0x%08X",
Kernel, NeedCopy, FlushData, UserPointer, Size, KernelPointer
);
/* Validate the arguemnts. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(UserPointer != gcvNULL);
gcmkVERIFY_ARGUMENT(KernelPointer != gcvNULL);
gcmkVERIFY_ARGUMENT(Size > 0);
/* Get a shortcut to the kernel pointer. */
pointer = * KernelPointer;
if (pointer != gcvNULL)
{
if (NeedCopy)
{
if (FlushData)
{
gcmkONERROR(gckOS_CopyToUserData(
Kernel->os, * KernelPointer, UserPointer, Size
));
}
}
else
{
/* Unmap record from kernel memory. */
gcmkONERROR(gckOS_UnmapUserPointer(
Kernel->os,
UserPointer,
Size,
* KernelPointer
));
}
/* Reset the kernel pointer. */
* KernelPointer = gcvNULL;
}
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckKERNEL_AllocateVirtualCommandBuffer(
IN gckKERNEL Kernel,
IN gctBOOL InUserSpace,
IN OUT gctSIZE_T * Bytes,
OUT gctPHYS_ADDR * Physical,
OUT gctPOINTER * Logical
)
{
gceSTATUS status;
gckOS os = Kernel->os;
gckVIRTUAL_COMMAND_BUFFER_PTR buffer;
gcmkHEADER_ARG("Os=0x%X InUserSpace=%d *Bytes=%lu",
os, InUserSpace, gcmOPT_VALUE(Bytes));
/* Verify the arguments. */
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Bytes != gcvNULL);
gcmkVERIFY_ARGUMENT(*Bytes > 0);
gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
gcmkONERROR(
gckOS_Allocate(
os,
sizeof(gckVIRTUAL_COMMAND_BUFFER),
(gctPOINTER)&buffer
));
gcmkONERROR(gckOS_ZeroMemory(buffer, sizeof(gckVIRTUAL_COMMAND_BUFFER)));
gcmkONERROR(
gckKERNEL_AllocateVirtualMemory(
Kernel,
gcvFALSE,
InUserSpace,
Bytes,
(gctPHYS_ADDR *)&buffer,
Logical
));
gcmkVERIFY_OK(gckOS_AcquireMutex(os, Kernel->virtualBufferLock, gcvINFINITE));
if (Kernel->virtualBufferHead == gcvNULL)
{
Kernel->virtualBufferHead =
Kernel->virtualBufferTail = buffer;
}
else
{
buffer->prev = Kernel->virtualBufferTail;
Kernel->virtualBufferTail->next = buffer;
Kernel->virtualBufferTail = buffer;
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Kernel->virtualBufferLock));
*Physical = buffer;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
gcmkVERIFY_OK(gckOS_Free(os, buffer));
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckKERNEL_DestroyVirtualCommandBuffer(
IN gckKERNEL Kernel,
IN gctSIZE_T Bytes,
IN gctPHYS_ADDR Physical,
IN gctPOINTER Logical
)
{
gckOS os;
gckKERNEL kernel;
gckVIRTUAL_COMMAND_BUFFER_PTR buffer = (gckVIRTUAL_COMMAND_BUFFER_PTR)Physical;
gcmkHEADER();
gcmkVERIFY_ARGUMENT(buffer != gcvNULL);
kernel = buffer->virtualBuffer.kernel;
os = kernel->os;
gcmkVERIFY_OK(gckOS_AcquireMutex(os, kernel->virtualBufferLock, gcvINFINITE));
if (buffer == kernel->virtualBufferHead)
{
if ((kernel->virtualBufferHead = buffer->next) == gcvNULL)
{
kernel->virtualBufferTail = gcvNULL;
}
}
else
{
buffer->prev->next = buffer->next;
if (buffer == kernel->virtualBufferTail)
{
kernel->virtualBufferTail = buffer->prev;
}
else
{
buffer->next->prev = buffer->prev;
}
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, kernel->virtualBufferLock));
gcmkVERIFY_OK(
gckKERNEL_FreeVirtualMemory(
Physical,
Logical,
gcvFALSE
));
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckKERNEL_AllocateVirtualMemory(
IN gckKERNEL Kernel,
IN gctBOOL NonPaged,
IN gctBOOL InUserSpace,
IN OUT gctSIZE_T * Bytes,
OUT gctPHYS_ADDR * Physical,
OUT gctPOINTER * Logical
)
{
gckOS os = Kernel->os;
gceSTATUS status;
gctPOINTER logical = gcvNULL;
gctSIZE_T pageCount;
gctSIZE_T bytes = *Bytes;
gckVIRTUAL_BUFFER_PTR buffer = gcvNULL;
gckMMU mmu = gcvNULL;
gctUINT32 allocFlag = 0;
gcmkHEADER_ARG("Os=0x%X InUserSpace=%d *Bytes=%lu",
os, InUserSpace, gcmOPT_VALUE(Bytes));
/* Verify the arguments. */
gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Bytes != gcvNULL);
gcmkVERIFY_ARGUMENT(*Bytes > 0);
gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
if (*Physical == gcvNULL)
{
gcmkONERROR(gckOS_Allocate(os,
sizeof(gckVIRTUAL_BUFFER),
(gctPOINTER)&buffer));
gcmkONERROR(gckOS_ZeroMemory(buffer, sizeof(gckVIRTUAL_BUFFER)));
}
else
{
buffer = *Physical;
}
buffer->bytes = bytes;
#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
allocFlag = gcvALLOC_FLAG_CACHEABLE;
#endif
if (NonPaged)
{
gcmkONERROR(gckOS_AllocateNonPagedMemory(
os,
InUserSpace,
allocFlag | gcvALLOC_FLAG_CONTIGUOUS,
&bytes,
&buffer->physical,
&logical
));
}
else
{
gcmkONERROR(gckOS_AllocatePagedMemoryEx(
os,
allocFlag | gcvALLOC_FLAG_NON_CONTIGUOUS,
bytes,
gcvNULL,
&buffer->physical
));
}
if (NonPaged)
{
gctSIZE_T pageSize;
gcmkONERROR(gckOS_GetPageSize(os, &pageSize));
pageCount = (bytes + pageSize - 1) / pageSize;
if (InUserSpace)
{
*Logical =
buffer->userLogical = logical;
}
else
{
*Logical =
buffer->kernelLogical = logical;
}
}
else
{
if (InUserSpace)
{
gcmkONERROR(gckOS_CreateUserVirtualMapping(os,
buffer->physical,
bytes,
&logical,
&pageCount));
*Logical =
buffer->userLogical = logical;
}
else
{
gcmkONERROR(gckOS_CreateKernelVirtualMapping(os,
buffer->physical,
bytes,
&logical,
&pageCount));
*Logical =
buffer->kernelLogical = logical;
}
}
buffer->pageCount = pageCount;
buffer->kernel = Kernel;
gcmkONERROR(gckOS_GetProcessID(&buffer->pid));
#if gcdPROCESS_ADDRESS_SPACE
gcmkONERROR(gckKERNEL_GetProcessMMU(Kernel, &mmu));
buffer->mmu = mmu;
#else
mmu = Kernel->mmu;
#endif
gcmkONERROR(gckMMU_AllocatePages(mmu,
pageCount,
&buffer->pageTable,
&buffer->gpuAddress));
#if gcdENABLE_TRUST_APPLICATION
if (Kernel->hardware->options.secureMode == gcvSECURE_IN_TA)
{
gcmkONERROR(gckKERNEL_MapInTrustApplicaiton(
Kernel,
logical,
buffer->physical,
buffer->gpuAddress,
pageCount
));
}
else
#endif
{
gcmkONERROR(gckOS_MapPagesEx(os,
Kernel->core,
buffer->physical,
pageCount,
buffer->gpuAddress,
buffer->pageTable,
gcvFALSE,
gcvSURF_TYPE_UNKNOWN
));
}
gcmkONERROR(gckMMU_Flush(mmu, gcvSURF_INDEX));
if (*Physical == gcvNULL)
*Physical = buffer;
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL,
"gpuAddress = %x pageCount = %d kernelLogical = %x userLogical=%x",
buffer->gpuAddress, buffer->pageCount,
buffer->kernelLogical, buffer->userLogical);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (buffer && buffer->gpuAddress)
{
gcmkVERIFY_OK(
gckMMU_FreePages(mmu, gcvFALSE, buffer->gpuAddress, buffer->pageTable, buffer->pageCount));
}
if (NonPaged && buffer->physical)
{
gcmkVERIFY_OK(gckOS_FreeNonPagedMemory(
os,
bytes,
buffer->physical,
logical
));
}
else
{
if (buffer && buffer->userLogical)
{
gcmkVERIFY_OK(
gckOS_DestroyUserVirtualMapping(os,
buffer->physical,
bytes,
(NonPaged ? 0 : buffer->userLogical)));
}
if (buffer && buffer->kernelLogical)
{
gcmkVERIFY_OK(
gckOS_DestroyKernelVirtualMapping(os,
buffer->physical,
bytes,
(NonPaged ? 0 : buffer->kernelLogical)));
}
if (buffer && buffer->physical)
{
gcmkVERIFY_OK(gckOS_FreePagedMemory(os, buffer->physical, bytes));
}
}
if (*Physical == gcvNULL)
gcmkVERIFY_OK(gckOS_Free(os, buffer));
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckKERNEL_FreeVirtualMemory(
IN gctPHYS_ADDR Physical,
IN gctPOINTER Logical,
IN gctBOOL NonPaged
)
{
gckOS os;
gckKERNEL kernel;
gckMMU mmu;
gckVIRTUAL_BUFFER_PTR buffer = (gckVIRTUAL_BUFFER_PTR)Physical;
gcmkHEADER();
gcmkVERIFY_ARGUMENT(buffer != gcvNULL);
kernel = buffer->kernel;
os = kernel->os;
#if gcdPROCESS_ADDRESS_SPACE
gcmkONERROR(gckKERNEL_GetProcessMMU(Kernel, &mmu));
#else
mmu = kernel->mmu;
#endif
if (!buffer->userLogical && !NonPaged)
{
gcmkVERIFY_OK(gckOS_DestroyKernelVirtualMapping(os,
buffer->physical,
buffer->bytes,
Logical));
}
gcmkVERIFY_OK(
gckMMU_FreePages(mmu, gcvFALSE, buffer->gpuAddress, buffer->pageTable, buffer->pageCount));
gcmkVERIFY_OK(gckOS_UnmapPages(os, buffer->pageCount, buffer->gpuAddress));
if (NonPaged)
{
gcmkVERIFY_OK(gckOS_FreeNonPagedMemory(
os,
buffer->bytes,
buffer->physical,
Logical
));
}
else
{
gcmkVERIFY_OK(gckOS_FreePagedMemory(os, buffer->physical, buffer->bytes));
}
gcmkVERIFY_OK(gckOS_Free(os, buffer));
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckKERNEL_GetGPUAddress(
IN gckKERNEL Kernel,
IN gctPOINTER Logical,
IN gctBOOL InUserSpace,
IN gctPHYS_ADDR Physical,
OUT gctUINT32 * Address
)
{
gckVIRTUAL_BUFFER_PTR buffer = Physical;
gctPOINTER start;
gcmkHEADER_ARG("Logical = %x InUserSpace=%d.", Logical, InUserSpace);
if (InUserSpace)
{
start = buffer->userLogical;
}
else
{
start = buffer->kernelLogical;
}
gcmkASSERT(Logical >= start
&& (Logical < (gctPOINTER)((gctUINT8_PTR)start + buffer->bytes)));
* Address = buffer->gpuAddress + (gctUINT32)((gctUINT8_PTR)Logical - (gctUINT8_PTR)start);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckKERNEL_QueryGPUAddress(
IN gckKERNEL Kernel,
IN gctUINT32 GpuAddress,
OUT gckVIRTUAL_COMMAND_BUFFER_PTR * Buffer
)
{
gckVIRTUAL_COMMAND_BUFFER_PTR buffer;
gctUINT32 start;
gceSTATUS status = gcvSTATUS_NOT_SUPPORTED;
gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, Kernel->virtualBufferLock, gcvINFINITE));
/* Walk all command buffers. */
for (buffer = Kernel->virtualBufferHead; buffer != gcvNULL; buffer = buffer->next)
{
start = (gctUINT32)buffer->virtualBuffer.gpuAddress;
if (GpuAddress >= start && GpuAddress <= (start - 1 + buffer->virtualBuffer.pageCount * 4096))
{
/* Find a range matched. */
*Buffer = buffer;
status = gcvSTATUS_OK;
break;
}
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->virtualBufferLock));
return status;
}
static void
gckQUEUE_Dequeue(
IN gckQUEUE LinkQueue
)
{
gcmkASSERT(LinkQueue->count == LinkQueue->size);
LinkQueue->count--;
LinkQueue->front = (LinkQueue->front + 1) % gcdLINK_QUEUE_SIZE;
}
void
gckQUEUE_Enqueue(
IN gckQUEUE LinkQueue,
IN gcuQUEUEDATA *Data
)
{
gcuQUEUEDATA * datas = LinkQueue->datas;
if (LinkQueue->count == LinkQueue->size)
{
gckQUEUE_Dequeue(LinkQueue);
}
gcmkASSERT(LinkQueue->count < LinkQueue->size);
LinkQueue->count++;
datas[LinkQueue->rear] = *Data;
LinkQueue->rear = (LinkQueue->rear + 1) % LinkQueue->size;
}
void
gckQUEUE_GetData(
IN gckQUEUE LinkQueue,
IN gctUINT32 Index,
OUT gcuQUEUEDATA ** Data
)
{
gcuQUEUEDATA * datas = LinkQueue->datas;
gcmkASSERT(Index >= 0 && Index < LinkQueue->size);
*Data = &datas[(Index + LinkQueue->front) % LinkQueue->size];
}
gceSTATUS
gckQUEUE_Allocate(
IN gckOS Os,
IN gckQUEUE Queue,
IN gctUINT32 Size
)
{
gceSTATUS status;
gcmkONERROR(gckOS_Allocate(
Os,
gcmSIZEOF(struct _gckLINKDATA) * Size,
(gctPOINTER *)&Queue->datas
));
Queue->size = Size;
return gcvSTATUS_OK;
OnError:
return status;
}
gceSTATUS
gckQUEUE_Free(
IN gckOS Os,
IN gckQUEUE Queue
)
{
if (Queue->datas)
{
gcmkVERIFY_OK(gckOS_Free(Os, (gctPOINTER)Queue->datas));
}
return gcvSTATUS_OK;
}
/******************************************************************************\
*************************** Pointer - ID translation ***************************
\******************************************************************************/
#define gcdID_TABLE_LENGTH 1024
typedef struct _gcsINTEGERDB * gckINTEGERDB;
typedef struct _gcsINTEGERDB
{
gckOS os;
gctPOINTER* table;
gctPOINTER mutex;
gctUINT32 tableLen;
gctUINT32 currentID;
gctUINT32 unused;
}
gcsINTEGERDB;
gceSTATUS
gckKERNEL_CreateIntegerDatabase(
IN gckKERNEL Kernel,
OUT gctPOINTER * Database
)
{
gceSTATUS status;
gckINTEGERDB database = gcvNULL;
gcmkHEADER_ARG("Kernel=0x%08X Datbase=0x%08X", Kernel, Database);
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(Database != gcvNULL);
/* Allocate a database. */
gcmkONERROR(gckOS_Allocate(
Kernel->os, gcmSIZEOF(gcsINTEGERDB), (gctPOINTER *)&database));
gcmkONERROR(gckOS_ZeroMemory(database, gcmSIZEOF(gcsINTEGERDB)));
/* Allocate a pointer table. */
gcmkONERROR(gckOS_Allocate(
Kernel->os, gcmSIZEOF(gctPOINTER) * gcdID_TABLE_LENGTH, (gctPOINTER *)&database->table));
gcmkONERROR(gckOS_ZeroMemory(database->table, gcmSIZEOF(gctPOINTER) * gcdID_TABLE_LENGTH));
/* Allocate a database mutex. */
gcmkONERROR(gckOS_CreateMutex(Kernel->os, &database->mutex));
/* Initialize. */
database->currentID = 0;
database->unused = gcdID_TABLE_LENGTH;
database->os = Kernel->os;
database->tableLen = gcdID_TABLE_LENGTH;
*Database = database;
gcmkFOOTER_ARG("*Database=0x%08X", *Database);
return gcvSTATUS_OK;
OnError:
/* Rollback. */
if (database)
{
if (database->table)
{
gcmkOS_SAFE_FREE(Kernel->os, database->table);
}
gcmkOS_SAFE_FREE(Kernel->os, database);
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckKERNEL_DestroyIntegerDatabase(
IN gckKERNEL Kernel,
IN gctPOINTER Database
)
{
gckINTEGERDB database = Database;
gcmkHEADER_ARG("Kernel=0x%08X Datbase=0x%08X", Kernel, Database);
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(Database != gcvNULL);
/* Destroy pointer table. */
gcmkOS_SAFE_FREE(Kernel->os, database->table);
/* Destroy database mutex. */
gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, database->mutex));
/* Destroy database. */
gcmkOS_SAFE_FREE(Kernel->os, database);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckKERNEL_AllocateIntegerId(
IN gctPOINTER Database,
IN gctPOINTER Pointer,
OUT gctUINT32 * Id
)
{
gceSTATUS status;
gckINTEGERDB database = Database;
gctUINT32 i, unused, currentID, tableLen;
gctPOINTER * table;
gckOS os = database->os;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Database=0x%08X Pointer=0x%08X", Database, Pointer);
gcmkVERIFY_ARGUMENT(Id != gcvNULL);
gcmkVERIFY_OK(gckOS_AcquireMutex(os, database->mutex, gcvINFINITE));
acquired = gcvTRUE;
if (database->unused < 1)
{
/* Extend table. */
gcmkONERROR(
gckOS_Allocate(os,
gcmSIZEOF(gctPOINTER) * (database->tableLen + gcdID_TABLE_LENGTH),
(gctPOINTER *)&table));
gcmkONERROR(gckOS_ZeroMemory(table + database->tableLen,
gcmSIZEOF(gctPOINTER) * gcdID_TABLE_LENGTH));
/* Copy data from old table. */
gckOS_MemCopy(table,
database->table,
database->tableLen * gcmSIZEOF(gctPOINTER));
gcmkOS_SAFE_FREE(os, database->table);
/* Update databse with new allocated table. */
database->table = table;
database->currentID = database->tableLen;
database->tableLen += gcdID_TABLE_LENGTH;
database->unused += gcdID_TABLE_LENGTH;
}
table = database->table;
currentID = database->currentID;
tableLen = database->tableLen;
unused = database->unused;
/* Connect id with pointer. */
table[currentID] = Pointer;
*Id = currentID + 1;
/* Update the currentID. */
if (--unused > 0)
{
for (i = 0; i < tableLen; i++)
{
if (++currentID >= tableLen)
{
/* Wrap to the begin. */
currentID = 0;
}
if (table[currentID] == gcvNULL)
{
break;
}
}
}
database->table = table;
database->currentID = currentID;
database->tableLen = tableLen;
database->unused = unused;
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
acquired = gcvFALSE;
gcmkFOOTER_ARG("*Id=%d", *Id);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckKERNEL_FreeIntegerId(
IN gctPOINTER Database,
IN gctUINT32 Id
)
{
gceSTATUS status;
gckINTEGERDB database = Database;
gckOS os = database->os;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Database=0x%08X Id=%d", Database, Id);
gcmkVERIFY_OK(gckOS_AcquireMutex(os, database->mutex, gcvINFINITE));
acquired = gcvTRUE;
if (!(Id > 0 && Id <= database->tableLen))
{
gcmkONERROR(gcvSTATUS_NOT_FOUND);
}
Id -= 1;
database->table[Id] = gcvNULL;
if (database->unused++ == 0)
{
database->currentID = Id;
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
acquired = gcvFALSE;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckKERNEL_QueryIntegerId(
IN gctPOINTER Database,
IN gctUINT32 Id,
OUT gctPOINTER * Pointer
)
{
gceSTATUS status;
gckINTEGERDB database = Database;
gctPOINTER pointer;
gckOS os = database->os;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Database=0x%08X Id=%d", Database, Id);
gcmkVERIFY_ARGUMENT(Pointer != gcvNULL);
gcmkVERIFY_OK(gckOS_AcquireMutex(os, database->mutex, gcvINFINITE));
acquired = gcvTRUE;
if (!(Id > 0 && Id <= database->tableLen))
{
gcmkONERROR(gcvSTATUS_NOT_FOUND);
}
Id -= 1;
pointer = database->table[Id];
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
acquired = gcvFALSE;
if (pointer)
{
*Pointer = pointer;
}
else
{
gcmkONERROR(gcvSTATUS_NOT_FOUND);
}
gcmkFOOTER_ARG("*Pointer=0x%08X", *Pointer);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
}
gcmkFOOTER();
return status;
}
gctUINT32
gckKERNEL_AllocateNameFromPointer(
IN gckKERNEL Kernel,
IN gctPOINTER Pointer
)
{
gceSTATUS status;
gctUINT32 name;
gctPOINTER database = Kernel->db->pointerDatabase;
gcmkHEADER_ARG("Kernel=0x%X Pointer=0x%X", Kernel, Pointer);
gcmkONERROR(
gckKERNEL_AllocateIntegerId(database, Pointer, &name));
gcmkFOOTER_ARG("name=%d", name);
return name;
OnError:
gcmkFOOTER();
return 0;
}
gctPOINTER
gckKERNEL_QueryPointerFromName(
IN gckKERNEL Kernel,
IN gctUINT32 Name
)
{
gceSTATUS status;
gctPOINTER pointer = gcvNULL;
gctPOINTER database = Kernel->db->pointerDatabase;
gcmkHEADER_ARG("Kernel=0x%X Name=%d", Kernel, Name);
/* Lookup in database to get pointer. */
gcmkONERROR(gckKERNEL_QueryIntegerId(database, Name, &pointer));
gcmkFOOTER_ARG("pointer=0x%X", pointer);
return pointer;
OnError:
gcmkFOOTER();
return gcvNULL;
}
gceSTATUS
gckKERNEL_DeleteName(
IN gckKERNEL Kernel,
IN gctUINT32 Name
)
{
gctPOINTER database = Kernel->db->pointerDatabase;
gcmkHEADER_ARG("Kernel=0x%X Name=0x%X", Kernel, Name);
/* Free name if exists. */
gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(database, Name));
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
***** Shared Buffer ************************************************************
*******************************************************************************/
/*******************************************************************************
**
** gckKERNEL_CreateShBuffer
**
** Create shared buffer.
** The shared buffer can be used across processes. Other process needs call
** gckKERNEL_MapShBuffer before use it.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctUINT32 Size
** Specify the shared buffer size.
**
** OUTPUT:
**
** gctSHBUF * ShBuf
** Pointer to hold return shared buffer handle.
*/
gceSTATUS
gckKERNEL_CreateShBuffer(
IN gckKERNEL Kernel,
IN gctUINT32 Size,
OUT gctSHBUF * ShBuf
)
{
gceSTATUS status;
gcsSHBUF_PTR shBuf = gcvNULL;
gcmkHEADER_ARG("Kernel=0x%X, Size=%u", Kernel, Size);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
if (Size == 0)
{
/* Invalid size. */
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
else if (Size > 1024)
{
/* Limite shared buffer size. */
gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
}
/* Create a shared buffer structure. */
gcmkONERROR(
gckOS_Allocate(Kernel->os,
sizeof (gcsSHBUF),
(gctPOINTER *)&shBuf));
/* Initialize shared buffer. */
shBuf->id = 0;
shBuf->reference = gcvNULL;
shBuf->size = Size;
shBuf->data = gcvNULL;
/* Allocate integer id for this shared buffer. */
gcmkONERROR(
gckKERNEL_AllocateIntegerId(Kernel->db->pointerDatabase,
shBuf,
&shBuf->id));
/* Allocate atom. */
gcmkONERROR(gckOS_AtomConstruct(Kernel->os, &shBuf->reference));
/* Set default reference count to 1. */
gcmkVERIFY_OK(gckOS_AtomSet(Kernel->os, shBuf->reference, 1));
/* Return integer id. */
*ShBuf = (gctSHBUF)(gctUINTPTR_T)shBuf->id;
gcmkFOOTER_ARG("*ShBuf=%u", shBuf->id);
return gcvSTATUS_OK;
OnError:
/* Error roll back. */
if (shBuf != gcvNULL)
{
if (shBuf->id != 0)
{
gcmkVERIFY_OK(
gckKERNEL_FreeIntegerId(Kernel->db->pointerDatabase,
shBuf->id));
}
gcmkOS_SAFE_FREE(Kernel->os, shBuf);
}
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_DestroyShBuffer
**
** Destroy shared buffer.
** This will decrease reference of specified shared buffer and do actual
** destroy when no reference on it.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctSHBUF ShBuf
** Specify the shared buffer to be destroyed.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckKERNEL_DestroyShBuffer(
IN gckKERNEL Kernel,
IN gctSHBUF ShBuf
)
{
gceSTATUS status;
gcsSHBUF_PTR shBuf;
gctINT32 oldValue = 0;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X ShBuf=%u",
Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL);
/* Acquire mutex. */
gcmkONERROR(
gckOS_AcquireMutex(Kernel->os,
Kernel->db->pointerDatabaseMutex,
gcvINFINITE));
acquired = gcvTRUE;
/* Find shared buffer structure. */
gcmkONERROR(
gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase,
(gctUINT32)(gctUINTPTR_T)ShBuf,
(gctPOINTER)&shBuf));
gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf);
/* Decrease the reference count. */
gckOS_AtomDecrement(Kernel->os, shBuf->reference, &oldValue);
if (oldValue == 1)
{
/* Free integer id. */
gcmkVERIFY_OK(
gckKERNEL_FreeIntegerId(Kernel->db->pointerDatabase,
shBuf->id));
/* Free atom. */
gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, shBuf->reference));
if (shBuf->data)
{
gcmkOS_SAFE_FREE(Kernel->os, shBuf->data);
shBuf->data = gcvNULL;
}
/* Free the shared buffer. */
gcmkOS_SAFE_FREE(Kernel->os, shBuf);
}
/* Release the mutex. */
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
acquired = gcvFALSE;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
}
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_MapShBuffer
**
** Map shared buffer into this process so that it can be used in this process.
** This will increase reference count on the specified shared buffer.
** Call gckKERNEL_DestroyShBuffer to dereference.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctSHBUF ShBuf
** Specify the shared buffer to be mapped.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckKERNEL_MapShBuffer(
IN gckKERNEL Kernel,
IN gctSHBUF ShBuf
)
{
gceSTATUS status;
gcsSHBUF_PTR shBuf;
gctINT32 oldValue = 0;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X ShBuf=%u",
Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL);
/* Acquire mutex. */
gcmkONERROR(
gckOS_AcquireMutex(Kernel->os,
Kernel->db->pointerDatabaseMutex,
gcvINFINITE));
acquired = gcvTRUE;
/* Find shared buffer structure. */
gcmkONERROR(
gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase,
(gctUINT32)(gctUINTPTR_T)ShBuf,
(gctPOINTER)&shBuf));
gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf);
/* Increase the reference count. */
gckOS_AtomIncrement(Kernel->os, shBuf->reference, &oldValue);
/* Release the mutex. */
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
acquired = gcvFALSE;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
}
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_WriteShBuffer
**
** Write user data into shared buffer.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctSHBUF ShBuf
** Specify the shared buffer to be written to.
**
** gctPOINTER UserData
** User mode pointer to hold the source data.
**
** gctUINT32 ByteCount
** Specify number of bytes to write. If this is larger than
** shared buffer size, gcvSTATUS_INVALID_ARGUMENT is returned.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckKERNEL_WriteShBuffer(
IN gckKERNEL Kernel,
IN gctSHBUF ShBuf,
IN gctPOINTER UserData,
IN gctUINT32 ByteCount
)
{
gceSTATUS status;
gcsSHBUF_PTR shBuf = gcvNULL;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X ShBuf=%u UserData=0x%X ByteCount=%u",
Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf, UserData, ByteCount);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL);
/* Acquire mutex. */
gcmkONERROR(
gckOS_AcquireMutex(Kernel->os,
Kernel->db->pointerDatabaseMutex,
gcvINFINITE));
acquired = gcvTRUE;
/* Find shared buffer structure. */
gcmkONERROR(
gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase,
(gctUINT32)(gctUINTPTR_T)ShBuf,
(gctPOINTER)&shBuf));
gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf);
if ((ByteCount > shBuf->size) ||
(ByteCount == 0) ||
(UserData == gcvNULL))
{
/* Exceeds buffer max size or invalid. */
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
if (shBuf->data == gcvNULL)
{
/* Allocate buffer data when first time write. */
gcmkONERROR(gckOS_Allocate(Kernel->os, ByteCount, &shBuf->data));
}
/* Copy data from user. */
gcmkONERROR(
gckOS_CopyFromUserData(Kernel->os,
shBuf->data,
UserData,
ByteCount));
/* Release the mutex. */
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
acquired = gcvFALSE;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (shBuf && shBuf->data)
{
gcmkOS_SAFE_FREE(Kernel->os, shBuf->data);
shBuf->data = gcvNULL;
}
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
}
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckKERNEL_ReadShBuffer
**
** Read data from shared buffer and copy to user pointer.
**
** INPUT:
**
** gckKERNEL Kernel
** Pointer to an gckKERNEL object.
**
** gctSHBUF ShBuf
** Specify the shared buffer to be read from.
**
** gctPOINTER UserData
** User mode pointer to save output data.
**
** gctUINT32 ByteCount
** Specify number of bytes to read.
** If this is larger than shared buffer size, only avaiable bytes are
** copied. If smaller, copy requested size.
**
** OUTPUT:
**
** gctUINT32 * BytesRead
** Pointer to hold how many bytes actually read from shared buffer.
*/
gceSTATUS
gckKERNEL_ReadShBuffer(
IN gckKERNEL Kernel,
IN gctSHBUF ShBuf,
IN gctPOINTER UserData,
IN gctUINT32 ByteCount,
OUT gctUINT32 * BytesRead
)
{
gceSTATUS status;
gcsSHBUF_PTR shBuf;
gctUINT32 bytes;
gctBOOL acquired = gcvFALSE;
gcmkHEADER_ARG("Kernel=0x%X ShBuf=%u UserData=0x%X ByteCount=%u",
Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf, UserData, ByteCount);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL);
/* Acquire mutex. */
gcmkONERROR(
gckOS_AcquireMutex(Kernel->os,
Kernel->db->pointerDatabaseMutex,
gcvINFINITE));
acquired = gcvTRUE;
/* Find shared buffer structure. */
gcmkONERROR(
gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase,
(gctUINT32)(gctUINTPTR_T)ShBuf,
(gctPOINTER)&shBuf));
gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf);
if (shBuf->data == gcvNULL)
{
*BytesRead = 0;
/* No data in shared buffer, skip copy. */
status = gcvSTATUS_SKIP;
goto OnError;
}
else if (ByteCount == 0)
{
/* Invalid size to read. */
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
/* Determine bytes to copy. */
bytes = (ByteCount < shBuf->size) ? ByteCount : shBuf->size;
/* Copy data to user. */
gcmkONERROR(
gckOS_CopyToUserData(Kernel->os,
shBuf->data,
UserData,
bytes));
/* Return copied size. */
*BytesRead = bytes;
/* Release the mutex. */
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
acquired = gcvFALSE;
gcmkFOOTER_ARG("*BytesRead=%u", bytes);
return gcvSTATUS_OK;
OnError:
if (acquired)
{
/* Release the mutex. */
gcmkVERIFY_OK(
gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
}
gcmkFOOTER();
return status;
}
/*******************************************************************************\
*************************** List Helper *****************************************
\*******************************************************************************/
static void
_ListAdd(
gcsLISTHEAD_PTR New,
gcsLISTHEAD_PTR Prev,
gcsLISTHEAD_PTR Next
)
{
Next->prev = New;
New->next = Next;
New->prev = Prev;
Prev->next = New;
}
void
_ListDel(
gcsLISTHEAD_PTR Prev,
gcsLISTHEAD_PTR Next
)
{
Next->prev = Prev;
Prev->next = Next;
}
void
gcsLIST_Init(
gcsLISTHEAD_PTR Node
)
{
Node->prev = Node;
Node->next = Node;
}
void
gcsLIST_Add(
gcsLISTHEAD_PTR New,
gcsLISTHEAD_PTR Head
)
{
_ListAdd(New, Head, Head->next);
}
void
gcsLIST_AddTail(
gcsLISTHEAD_PTR New,
gcsLISTHEAD_PTR Head
)
{
_ListAdd(New, Head->prev, Head);
}
void
gcsLIST_Del(
gcsLISTHEAD_PTR Node
)
{
_ListDel(Node->prev, Node->next);
}
gctBOOL
gcsLIST_Empty(
gcsLISTHEAD_PTR Head
)
{
return Head->next == Head;
}
/*******************************************************************************\
********************************* Fence *****************************************
\*******************************************************************************/
gceSTATUS
gckFENCE_Create(
IN gckOS Os,
IN gckKERNEL Kernel,
OUT gckFENCE * Fence
)
{
gceSTATUS status;
gckFENCE fence = gcvNULL;
gctSIZE_T pageSize = 4096;
gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcsFENCE), (gctPOINTER *)&fence));
gcmkONERROR(gckOS_ZeroMemory(fence, gcmSIZEOF(gcsFENCE)));
gcmkONERROR(gckOS_CreateMutex(Os, (gctPOINTER *)&fence->mutex));
fence->kernel = Kernel;
#if USE_KERNEL_VIRTUAL_BUFFERS
if (Kernel->virtualCommandBuffer)
{
gcmkONERROR(gckKERNEL_AllocateVirtualMemory(
Kernel,
gcvFALSE,
gcvFALSE,
&pageSize,
&fence->physical,
&fence->logical
));
gcmkONERROR(gckKERNEL_GetGPUAddress(
Kernel,
fence->logical,
gcvFALSE,
fence->physical,
&fence->address
));
}
else
#endif
{
gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS;
#if gcdENABLE_CACHEABLE_COMMAND_BUFFER
allocFlag |= gcvALLOC_FLAG_CACHEABLE;
#endif
gcmkONERROR(gckOS_AllocateNonPagedMemory(
Os,
gcvFALSE,
allocFlag,
&pageSize,
&fence->physical,
&fence->logical
));
gcmkONERROR(gckHARDWARE_ConvertLogical(
Kernel->hardware,
fence->logical,
gcvFALSE,
&fence->address
));
gcmkONERROR(gckMMU_FillFlatMapping(
Kernel->mmu, fence->address, pageSize
));
}
gcsLIST_Init(&fence->waitingList);
*Fence = fence;
return gcvSTATUS_OK;
OnError:
if (fence)
{
gckFENCE_Destory(Os, fence);
}
return status;
}
gceSTATUS
gckFENCE_Destory(
IN gckOS Os,
OUT gckFENCE Fence
)
{
if (Fence->mutex)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(Os, Fence->mutex));
}
if (Fence->logical)
{
#if USE_KERNEL_VIRTUAL_BUFFERS
if (Fence->kernel->virtualCommandBuffer)
{
gcmkVERIFY_OK(gckKERNEL_FreeVirtualMemory(
Fence->physical,
Fence->logical,
gcvFALSE
));
}
else
#endif
{
gcmkVERIFY_OK(gckOS_FreeNonPagedMemory(
Os,
4096,
Fence->physical,
Fence->logical
));
}
}
gcmkOS_SAFE_FREE(Os, Fence);
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckFENCE_Signal
**
** Signal all completed nodes.
**
**
*/
gceSTATUS
gckFENCE_Signal(
IN gckOS Os,
IN gckFENCE Fence
)
{
gcsLISTHEAD_PTR list = &Fence->waitingList;
gcsLISTHEAD_PTR nodeHead, nodeTemp;
gckFENCE_SYNC sync;
gckOS os = Os;
gctUINT64 stamp = *(gctUINT64 *)Fence->logical;
gcmkVERIFY_OK(gckOS_AcquireMutex(os, Fence->mutex, gcvINFINITE));
gcmkLIST_FOR_EACH_SAFE(nodeHead, nodeTemp, list)
{
sync = gcmCONTAINEROF(nodeHead, _gcsFENCE_SYNC, head);
/* Signal all nodes which are complete. */
if (sync->commitStamp <= stamp && sync->inList)
{
/* Signal. */
gckOS_Signal(os, sync->signal, gcvTRUE);
/* Remove from wait list. */
gcsLIST_Del(nodeHead);
/* Mark node not in waiting list. */
sync->inList = gcvFALSE;
}
}
gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Fence->mutex));
return gcvSTATUS_OK;
}
gceSTATUS
gckDEVICE_Construct(
IN gckOS Os,
OUT gckDEVICE * Device
)
{
gceSTATUS status;
gckDEVICE device;
gctUINT i;
gcmkHEADER();
gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcsDEVICE), (gctPOINTER *)&device));
for (i = 0; i < gcvCORE_COUNT; i++)
{
device->coreInfoArray[i].type = gcvHARDWARE_INVALID;
}
device->defaultHwType = gcvHARDWARE_INVALID;
gckOS_ZeroMemory(device, gcmSIZEOF(gcsDEVICE));
gcmkONERROR(gckOS_CreateMutex(Os, &device->stuckDumpMutex));
gcmkONERROR(gckOS_CreateMutex(Os, &device->commitMutex));
device->os = Os;
*Device = device;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (device != gcvNULL)
{
gckDEVICE_Destroy(Os, device);
}
gcmkFOOTER();
return status;
}
gceSTATUS
gckDEVICE_AddCore(
IN gckDEVICE Device,
IN gceCORE Core,
IN gctUINT ChipID,
IN gctPOINTER Context,
IN gckKERNEL * Kernel
)
{
gceSTATUS status;
gcsCORE_INFO * info = Device->coreInfoArray;
gceHARDWARE_TYPE type = (gceHARDWARE_TYPE)((gctUINT)gcvHARDWARE_INVALID);
gctUINT32 index = Device->coreNum;
gctUINT32 i;
gcsCORE_LIST *coreList;
gceHARDWARE_TYPE kernelType;
gceHARDWARE_TYPE defaultHwType;
gckKERNEL kernel;
gcmkASSERT(Device->coreNum < gcvCORE_COUNT);
if (Core >= gcvCORE_MAJOR && Core <= gcvCORE_3D_MAX)
{
/* Chip ID is only used for 3D cores. */
if (ChipID == gcvCHIP_ID_DEFAULT)
{
/* Apply default chipID if it is not set. */
ChipID = Core;
}
}
/* Construct gckKERNEL for this core. */
gcmkONERROR(gckKERNEL_Construct(
Device->os, Core, ChipID, Context, Device, Device->database, Kernel));
kernel = *Kernel;
if (Device->database == gcvNULL)
{
Device->database = kernel->db;
}
kernelType = _GetHardwareType(kernel);
if (kernelType >= gcvHARDWARE_NUM_TYPES)
{
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
info[index].type = kernelType;
info[index].core = Core;
info[index].kernel = kernel;
info[index].chipID = ChipID;
if (index == 0)
{
/* First core, map all type/core to it. */
for (; type != gcvHARDWARE_NUM_TYPES; type = (gceHARDWARE_TYPE)((gctUINT)type + 1))
{
Device->map[type].num = 0;
for (i = 0 ; i < 4; i++)
{
Device->map[type].kernels[i] = kernel;
}
}
}
/* Get core list of this type. */
coreList = &Device->map[kernelType];
/* Setup gceHARDWARE_TYPE to gceCORE mapping. */
coreList->kernels[coreList->num++] = kernel;
defaultHwType = kernelType;
if (kernelType == gcvHARDWARE_3D2D)
{
coreList = &Device->map[gcvHARDWARE_3D];
coreList->kernels[coreList->num++] = kernel;
defaultHwType = gcvHARDWARE_3D;
}
/* Advance total core number. */
Device->coreNum++;
/* Default HW type was chosen: 3D > 2D > VG */
if (Device->defaultHwType == gcvHARDWARE_INVALID)
{
Device->defaultHwType = defaultHwType;
}
else if (Device->defaultHwType > defaultHwType)
{
Device->defaultHwType = defaultHwType;
}
return gcvSTATUS_OK;
OnError:
return status;
}
gceSTATUS
gckDEVICE_ChipInfo(
IN gckDEVICE Device,
IN gcsHAL_INTERFACE_PTR Interface
)
{
gctUINT i;
gcsCORE_INFO * info = Device->coreInfoArray;
for (i = 0; i < Device->coreNum; i++)
{
Interface->u.ChipInfo.types[i] = info[i].type;
Interface->u.ChipInfo.ids[i] = info[i].chipID;
}
Interface->u.ChipInfo.count = Device->coreNum;
return gcvSTATUS_OK;
}
gceSTATUS
gckDEVICE_Version(
IN gckDEVICE Device,
IN gcsHAL_INTERFACE_PTR Interface
)
{
Interface->u.Version.major = gcvVERSION_MAJOR;
Interface->u.Version.minor = gcvVERSION_MINOR;
Interface->u.Version.patch = gcvVERSION_PATCH;
Interface->u.Version.build = gcvVERSION_BUILD;
#if gcmIS_DEBUG(gcdDEBUG_TRACE)
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL,
"KERNEL version %d.%d.%d build %u",
gcvVERSION_MAJOR, gcvVERSION_MINOR,
gcvVERSION_PATCH, gcvVERSION_BUILD);
#endif
return gcvSTATUS_OK;
}
gceSTATUS
gckDEVICE_Destroy(
IN gckOS Os,
IN gckDEVICE Device
)
{
gctINT i;
gcsCORE_INFO * info = Device->coreInfoArray;
for (i = Device->coreNum - 1; i >= 0 ; i--)
{
if (info[i].kernel != gcvNULL)
{
gckKERNEL_Destroy(info[i].kernel);
}
}
if (Device->commitMutex)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(Os, Device->commitMutex));
}
if (Device->stuckDumpMutex)
{
gcmkVERIFY_OK(gckOS_DeleteMutex(Os, Device->stuckDumpMutex));
}
gcmkOS_SAFE_FREE(Os, Device);
return gcvSTATUS_OK;
}
static gceSTATUS
gckDEVICE_SetTimeOut(
IN gckDEVICE Device,
IN gcsHAL_INTERFACE_PTR Interface
)
{
#if gcdGPU_TIMEOUT
gckKERNEL kernel;
gctUINT i;
gceHARDWARE_TYPE type = Interface->hardwareType;
gcsCORE_LIST *coreList;
coreList = &Device->map[type];
for (i = 0; i < coreList->num; i++)
{
kernel = coreList->kernels[i];
kernel->timeOut = Interface->u.SetTimeOut.timeOut;
}
#endif
return gcvSTATUS_OK;
}
gceSTATUS
gckDEVICE_Dispatch(
IN gckDEVICE Device,
IN gcsHAL_INTERFACE_PTR Interface
)
{
gceSTATUS status = gcvSTATUS_NOT_SUPPORTED;
gckKERNEL kernel;
gceHARDWARE_TYPE type = Interface->hardwareType;
gctUINT32 coreIndex = Interface->coreIndex;
switch (Interface->command)
{
case gcvHAL_CHIP_INFO:
status = gckDEVICE_ChipInfo(Device, Interface);
break;
case gcvHAL_VERSION:
status = gckDEVICE_Version(Device, Interface);
break;
case gcvHAL_SET_TIMEOUT:
status = gckDEVICE_SetTimeOut(Device, Interface);
break;
default:
status = gcvSTATUS_NOT_SUPPORTED;
break;
}
if (gcmIS_SUCCESS(status))
{
/* Dispatch handled in this layer. */
Interface->status = status;
}
else
{
/* Need go through gckKERNEL dispatch. */
kernel = Device->map[type].kernels[coreIndex];
#if gcdENABLE_VG
if (kernel->vg)
{
status = gckVGKERNEL_Dispatch(kernel, gcvTRUE, Interface);
}
else
#endif
{
status = gckKERNEL_Dispatch(kernel, Device, gcvTRUE, Interface);
}
/* Interface->status is handled in gckKERNEL_Dispatch(). */
}
return status;
}
gceSTATUS
gckDEVICE_GetMMU(
IN gckDEVICE Device,
IN gceHARDWARE_TYPE Type,
IN gckMMU *Mmu
)
{
gcmkHEADER();
gcmkVERIFY_ARGUMENT(Type < gcvHARDWARE_NUM_TYPES);
*Mmu = Device->mmus[Type];
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckDEVICE_SetMMU(
IN gckDEVICE Device,
IN gceHARDWARE_TYPE Type,
IN gckMMU Mmu
)
{
gcmkHEADER();
gcmkVERIFY_ARGUMENT(Type < gcvHARDWARE_NUM_TYPES);
Device->mmus[Type] = Mmu;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckDEVICE_QueryGPUAddress
**
** Search GPUAddress in other core's address space, whose type is same as current
** core. It is used to find correct command buffer which is shared by mulitple
** core.
**
*/
gceSTATUS
gckDEVICE_QueryGPUAddress(
IN gckDEVICE Device,
IN gckKERNEL Kernel,
IN gctUINT32 GPUAddress,
OUT gckVIRTUAL_COMMAND_BUFFER_PTR * Buffer
)
{
gceSTATUS status = gcvSTATUS_NOT_FOUND;
gctUINT i;
gceHARDWARE_TYPE kernelType;
kernelType = _GetHardwareType(Kernel);
if (Device != gcvNULL)
{
for (i = 0; i < Device->coreNum; i++)
{
if (Device->coreInfoArray[i].type == kernelType)
{
/* Search other core's command buffer list whose type is same. */
status = gckKERNEL_QueryGPUAddress(
Device->coreInfoArray[i].kernel, GPUAddress, Buffer);
if (gcmIS_SUCCESS(status))
{
break;
}
}
}
}
else
{
status = gckKERNEL_QueryGPUAddress(Kernel, GPUAddress, Buffer);
}
return status;
}
#if gcdENABLE_TRUST_APPLICATION
gceSTATUS
gckKERNEL_MapInTrustApplicaiton(
IN gckKERNEL Kernel,
IN gctPOINTER Logical,
IN gctPHYS_ADDR Physical,
IN gctUINT32 GPUAddress,
IN gctSIZE_T PageCount
)
{
gceSTATUS status;
gctUINT32 * physicalArrayLogical = gcvNULL;
gctSIZE_T bytes;
gctPOINTER logical = Logical;
gctUINT32 i;
gctSIZE_T pageSize;
gctUINT32 pageMask;
gcmkHEADER();
gcmkVERIFY_OK(gckOS_GetPageSize(Kernel->os, &pageSize));
pageMask = (gctUINT32)pageSize - 1;
bytes = PageCount * gcmSIZEOF(gctUINT32);
gcmkONERROR(gckOS_Allocate(
Kernel->os,
bytes,
(gctPOINTER *)&physicalArrayLogical
));
/* Fill in physical array. */
for (i = 0; i < PageCount; i++)
{
gctPHYS_ADDR_T phys;
status = gckOS_PhysicalToPhysicalAddress(
Kernel->os,
Physical,
i * 4096,
&phys
);
if (status == gcvSTATUS_NOT_SUPPORTED)
{
gcmkONERROR(gckOS_GetPhysicalAddress(
Kernel->os,
logical,
&phys
));
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Kernel->os, phys, &phys));
}
phys &= ~pageMask;
gcmkSAFECASTPHYSADDRT(physicalArrayLogical[i], phys);
logical = (gctUINT8_PTR)logical + 4096;
}
gcmkONERROR(gckKERNEL_SecurityMapMemory(
Kernel,
physicalArrayLogical,
0,
(gctUINT32)PageCount,
&GPUAddress
));
gcmkVERIFY_OK(gckOS_Free(
Kernel->os,
physicalArrayLogical
))
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if(physicalArrayLogical != gcvNULL)
gcmkVERIFY_OK(gckOS_Free(
Kernel->os,
(gctPOINTER)physicalArrayLogical
));
gcmkFOOTER();
return status;
}
#endif
/*******************************************************************************
***** Test Code ****************************************************************
*******************************************************************************/