Extract Verisilicon_SW_GAL3D_Unified_Src_kernel_drv_6_4_2_20200911.tgz
diff --git a/Kbuild b/Kbuild new file mode 100644 index 0000000..ceb6a88 --- /dev/null +++ b/Kbuild
@@ -0,0 +1,308 @@ +############################################################################## +# +# The MIT License (MIT) +# +# Copyright (c) 2014 - 2020 Vivante Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +# +############################################################################## +# +# The GPL License (GPL) +# +# Copyright (C) 2014 - 2020 Vivante Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +############################################################################## +# +# Note: This software is released under dual MIT and GPL licenses. A +# recipient may use this file under the terms of either the MIT license or +# GPL License. If you wish to use only one license not the other, you can +# indicate your decision by deleting one of the above license notices in your +# version of this file. +# +############################################################################## + + +# +# Linux build file for kernel HAL driver. +# + +include $(AQROOT)/config + +soc_vendor := $(firstword $(subst -, ,$(SOC_PLATFORM))) +soc_board := $(lastword $(subst -, ,$(SOC_PLATFORM))) + +KERNEL_DIR ?= $(TOOL_DIR)/kernel + +OS_KERNEL_DIR := hal/os/linux/kernel +ARCH_KERNEL_DIR := hal/kernel/arch +ARCH_VG_KERNEL_DIR := hal/kernel/archvg +HAL_KERNEL_DIR := hal/kernel +TA_DIR := hal/security_v1 +HOST := $(shell hostname) + +MODULE_NAME ?= galcore +CUSTOMER_ALLOCATOR_OBJS ?= +ALLOCATOR_ARRAY_H_LOCATION ?= $(OS_KERNEL_DIR)/allocator/default/ +SOC_PLATFORM_LOCATION ?= $(OS_KERNEL_DIR)/platform/$(soc_vendor) + +# Set common platform driver source +SOC_PLATFORM_OBJS ?= $(SOC_PLATFORM_LOCATION)/gc_hal_kernel_platform_$(soc_board).o +# Include platform config if exists. +-include $(AQROOT)/$(SOC_PLATFORM_LOCATION)/gc_hal_kernel_platform_$(soc_board).config + +ifeq ($(CONFIG_KASAN),) +EXTRA_CFLAGS += -Werror -Wno-implicit-fallthrough +endif + +OBJS := $(OS_KERNEL_DIR)/gc_hal_kernel_device.o \ + $(OS_KERNEL_DIR)/gc_hal_kernel_linux.o \ + $(OS_KERNEL_DIR)/gc_hal_kernel_math.o \ + $(OS_KERNEL_DIR)/gc_hal_kernel_os.o \ + $(OS_KERNEL_DIR)/gc_hal_kernel_debugfs.o \ + $(OS_KERNEL_DIR)/gc_hal_kernel_allocator.o \ + $(OS_KERNEL_DIR)/allocator/default/gc_hal_kernel_allocator_user_memory.o \ + $(OS_KERNEL_DIR)/allocator/default/gc_hal_kernel_allocator_dma.o \ + $(OS_KERNEL_DIR)/allocator/default/gc_hal_kernel_allocator_gfp.o \ + $(OS_KERNEL_DIR)/allocator/default/gc_hal_kernel_allocator_reserved_mem.o \ + $(OS_KERNEL_DIR)/gc_hal_kernel_driver.o + +# Source files for soc platform board +OBJS += $(SOC_PLATFORM_OBJS) + +ifneq ($(CONFIG_DMA_SHARED_BUFFER),) +OBJS += $(OS_KERNEL_DIR)/allocator/default/gc_hal_kernel_allocator_dmabuf.o +endif + +ifneq ($(CONFIG_IOMMU_SUPPORT),) +OBJS += $(OS_KERNEL_DIR)/gc_hal_kernel_iommu.o +endif + +ifneq ($(CONFIG_DRM),) +OBJS += $(OS_KERNEL_DIR)/gc_hal_kernel_drm.o +endif + +OBJS += $(HAL_KERNEL_DIR)/gc_hal_kernel.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_command.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_db.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_debug.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_event.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_heap.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_mmu.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_video_memory.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_power.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_security_v1.o + +OBJS += $(ARCH_KERNEL_DIR)/gc_hal_kernel_context.o \ + $(ARCH_KERNEL_DIR)/gc_hal_kernel_hardware.o \ + $(ARCH_KERNEL_DIR)/gc_hal_kernel_hardware_func.o \ + $(ARCH_KERNEL_DIR)/gc_hal_kernel_hardware_async_fe.o \ + $(ARCH_KERNEL_DIR)/gc_hal_kernel_hardware_mc_fe.o \ + $(ARCH_KERNEL_DIR)/gc_hal_kernel_hardware_waitlink_fe.o + +ifeq ($(VIVANTE_ENABLE_3D),1) +OBJS += $(ARCH_KERNEL_DIR)/gc_hal_kernel_recorder.o +endif + + +ifneq ($(CONFIG_SYNC),) +EXTRA_CFLAGS += -Idrivers/staging/android +EXTRA_CFLAGS += -DgcdLINUX_SYNC_FILE=1 + +OBJS += $(OS_KERNEL_DIR)/gc_hal_kernel_sync.o +else + ifneq ($(CONFIG_SYNC_FILE),) + EXTRA_CFLAGS += -DgcdLINUX_SYNC_FILE=1 + OBJS += $(OS_KERNEL_DIR)/gc_hal_kernel_sync.o + endif +endif + +ifeq ($(SECURITY),1) +OBJS += $(OS_KERNEL_DIR)/gc_hal_kernel_security_channel.o \ + $(HAL_KERNEL_DIR)/gc_hal_kernel_security.o +endif + +ifneq ($(CUSTOMER_ALLOCATOR_OBJS),) +OBJS += $(CUSTOMER_ALLOCATOR_OBJS) +endif + +OBJS += $(OS_KERNEL_DIR)/gc_hal_kernel_security_channel_emulator.o \ + $(TA_DIR)/gc_hal_ta.o \ + $(TA_DIR)/gc_hal_ta_hardware.o \ + $(TA_DIR)/gc_hal_ta_mmu.o \ + $(TA_DIR)/os/emulator/gc_hal_ta_emulator.o + +ifeq ($(KERNELRELEASE),) + +.PHONY: all clean install + +# Define targets. +all: + @$(MAKE) V=$(V) ARCH=$(ARCH_TYPE) -C $(KERNEL_DIR) M=`pwd` modules + +clean: + @rm -rf $(OBJS) + @rm -rf modules.order Module.symvers .tmp_versions + @find $(AQROOT) -name ".gc_*.cmd" | xargs rm -f + +install: all + @mkdir -p $(SDK_DIR)/drivers + @cp $(MODULE_NAME).ko $(SDK_DIR)/drivers + +else + + +EXTRA_CFLAGS += -DLINUX -DDRIVER + +ifeq ($(FLAREON),1) +EXTRA_CFLAGS += -DFLAREON +endif + +ifeq ($(DEBUG),1) +EXTRA_CFLAGS += -DDBG=1 -DDEBUG -D_DEBUG +else +EXTRA_CFLAGS += -DDBG=0 +endif + +ifeq ($(NO_DMA_COHERENT),1) +EXTRA_CFLAGS += -DNO_DMA_COHERENT +endif + +ifeq ($(CONFIG_DOVE_GPU),1) +EXTRA_CFLAGS += -DCONFIG_DOVE_GPU=1 +endif + +ifneq ($(USE_PLATFORM_DRIVER),0) +EXTRA_CFLAGS += -DUSE_PLATFORM_DRIVER=1 +else +EXTRA_CFLAGS += -DUSE_PLATFORM_DRIVER=0 +endif + +EXTRA_CFLAGS += -DVIVANTE_PROFILER=1 +EXTRA_CFLAGS += -DVIVANTE_PROFILER_CONTEXT=1 + +ifeq ($(ENABLE_GPU_CLOCK_BY_DRIVER),1) +EXTRA_CFLAGS += -DENABLE_GPU_CLOCK_BY_DRIVER=1 +else +EXTRA_CFLAGS += -DENABLE_GPU_CLOCK_BY_DRIVER=0 +endif + +ifeq ($(USE_NEW_LINUX_SIGNAL),1) +EXTRA_CFLAGS += -DUSE_NEW_LINUX_SIGNAL=1 +else +EXTRA_CFLAGS += -DUSE_NEW_LINUX_SIGNAL=0 +endif + +ifeq ($(USE_LINUX_PCIE),1) +EXTRA_CFLAGS += -DUSE_LINUX_PCIE=1 +else +EXTRA_CFLAGS += -DUSE_LINUX_PCIE=0 +endif + +ifeq ($(CACHE_FUNCTION_UNIMPLEMENTED),1) +EXTRA_CFLAGS += -DgcdCACHE_FUNCTION_UNIMPLEMENTED=1 +else +EXTRA_CFLAGS += -DgcdCACHE_FUNCTION_UNIMPLEMENTED=0 +endif + +ifeq ($(VIVANTE_ENABLE_3D),0) +EXTRA_CFLAGS += -DgcdENABLE_3D=0 +else +EXTRA_CFLAGS += -DgcdENABLE_3D=1 +endif + +EXTRA_CFLAGS += -DgcdENABLE_2D=0 + +EXTRA_CFLAGS += -DgcdENABLE_VG=0 + +ifeq ($(USE_BANK_ALIGNMENT),1) + EXTRA_CFLAGS += -DgcdENABLE_BANK_ALIGNMENT=1 + ifneq ($(BANK_BIT_START),0) + ifneq ($(BANK_BIT_END),0) + EXTRA_CFLAGS += -DgcdBANK_BIT_START=$(BANK_BIT_START) + EXTRA_CFLAGS += -DgcdBANK_BIT_END=$(BANK_BIT_END) + endif + endif + + ifneq ($(BANK_CHANNEL_BIT),0) + EXTRA_CFLAGS += -DgcdBANK_CHANNEL_BIT=$(BANK_CHANNEL_BIT) + endif +endif + +ifeq ($(FPGA_BUILD),1) +EXTRA_CFLAGS += -DgcdFPGA_BUILD=1 +else +EXTRA_CFLAGS += -DgcdFPGA_BUILD=0 +endif + +ifeq ($(SECURITY),1) +EXTRA_CFLAGS += -DgcdSECURITY=1 +endif + +ifneq ($(CONFIG_DRM),) + ifneq ($(CONFIG_ANDROID),) + EXTRA_CFLAGS += -DgcdENABLE_DRM=$(VIVANTE_ENABLE_DRM) + else + EXTRA_CFLAGS += -DgcdENABLE_DRM=0 + endif +else +EXTRA_CFLAGS += -DgcdENABLE_DRM=0 +endif + +EXTRA_CFLAGS += -I$(AQROOT)/hal/kernel/inc +EXTRA_CFLAGS += -I$(AQROOT)/hal/kernel +EXTRA_CFLAGS += -I$(AQROOT)/hal/kernel/arch +EXTRA_CFLAGS += -I$(AQROOT)/hal/kernel/inc +EXTRA_CFLAGS += -I$(AQROOT)/hal/os/linux/kernel +EXTRA_CFLAGS += -I$(AQROOT)/$(ALLOCATOR_ARRAY_H_LOCATION) +EXTRA_CFLAGS += -I$(AQROOT)/hal/security_v1/ +EXTRA_CFLAGS += -I$(AQROOT)/$(SOC_PLATFORM_LOCATION) + +ifneq ($(CONFIG_ARM),) +EXTRA_CFLAGS += -Iarch/arm/mm +endif + + +ifeq ($(VIVANTE_ENABLE_DRM),1) +EXTRA_CFLAGS += -I$(AQROOT)/driver/X/libdrm-2.4.91/include/drm +endif + +EXTRA_CFLAGS += -DHOST=\"$(HOST)\" + +EXTRA_CFLAGS += -DgcdENABLE_TRUST_APPLICATION=1 + +obj-m = $(MODULE_NAME).o + +$(MODULE_NAME)-objs = $(OBJS) + +endif
diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2185128 --- /dev/null +++ b/Makefile
@@ -0,0 +1,82 @@ +############################################################################## +# +# The MIT License (MIT) +# +# Copyright (c) 2014 - 2020 Vivante Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +# +############################################################################## +# +# The GPL License (GPL) +# +# Copyright (C) 2014 - 2020 Vivante Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +############################################################################## +# +# Note: This software is released under dual MIT and GPL licenses. A +# recipient may use this file under the terms of either the MIT license or +# GPL License. If you wish to use only one license not the other, you can +# indicate your decision by deleting one of the above license notices in your +# version of this file. +# +############################################################################## + + +AQROOT=$(CURDIR) +AQARCH=$(AQROOT)/arch/XAQ2 + +include $(AQROOT)/config + + +export AQROOT AQARCH + + +ARCH_TYPE ?= arm +KERNEL_DIR ?= /home/software/Linux/linux-2.6.21-arm1 +CROSS_COMPILE ?= /home/software/Linux/arm-2007q1/bin/arm-none-linux-gnueabi- +SDK_DIR ?= $(AQROOT)/build/sdk + +export KERNEL_DIR CROSS_COMPILE SDK_DIR + +.PHONY: all clean install + +all: + @make -f Kbuild + +clean: + @make -f Kbuild ARCH=$(ARCH_TYPE) clean + +install: + @make -f Kbuild install +
diff --git a/config b/config new file mode 100644 index 0000000..271dbdc --- /dev/null +++ b/config
@@ -0,0 +1,71 @@ +############################################################################## +# +# The MIT License (MIT) +# +# Copyright (c) 2014 - 2020 Vivante Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +# +############################################################################## +# +# The GPL License (GPL) +# +# Copyright (C) 2014 - 2020 Vivante Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +############################################################################## +# +# Note: This software is released under dual MIT and GPL licenses. A +# recipient may use this file under the terms of either the MIT license or +# GPL License. If you wish to use only one license not the other, you can +# indicate your decision by deleting one of the above license notices in your +# version of this file. +# +############################################################################## + + +ARCH_TYPE ?= arm +SDK_DIR ?= $(AQROOT)/build/sdk +VIVANTE_ENABLE_3D ?= 1 +VIVANTE_ENABLE_2D ?= 1 +VIVANTE_ENABLE_VG ?= 1 +VIVANTE_ENABLE_DRM ?= 0 +NO_DMA_COHERENT ?= 0 +USE_PLATFORM_DRIVER ?= 1 +ENABLE_GPU_CLOCK_BY_DRIVER ?= 1 +CACHE_FUNCTION_UNIMPLEMENTED ?= 0 +USE_BANK_ALIGNMENT ?= 0 +BANK_BIT_START ?= 0 +BANK_BIT_END ?= 0 +BANK_CHANNEL_BIT ?= 0 +SECURITY ?= 0 +SOC_PLATFORM ?= default
diff --git a/hal/kernel/arch/gc_hal_kernel_context.c b/hal/kernel/arch/gc_hal_kernel_context.c new file mode 100644 index 0000000..a059ad9 --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_context.c
@@ -0,0 +1,5269 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal.h" +#include "gc_hal_kernel.h" +#include "gc_hal_kernel_context.h" +#include "gc_hal_kernel_buffer.h" + +/******************************************************************************\ +******************************** Debugging Macro ******************************* +\******************************************************************************/ + +/* Zone used for header/footer. */ +#define _GC_OBJ_ZONE gcvZONE_HARDWARE + + +/******************************************************************************\ +************************** Context State Buffer Helpers ************************ +\******************************************************************************/ + +#define _STATE(reg) \ + _State(\ + Context, index, \ + reg ## _Address >> 2, \ + reg ## _ResetValue, \ + reg ## _Count, \ + gcvFALSE, gcvFALSE \ + ) + +#define _STATE_COUNT(reg, count) \ + _State(\ + Context, index, \ + reg ## _Address >> 2, \ + reg ## _ResetValue, \ + count, \ + gcvFALSE, gcvFALSE \ + ) + +#define _STATE_COUNT_OFFSET(reg, offset, count) \ + _State(\ + Context, index, \ + (reg ## _Address >> 2) + offset, \ + reg ## _ResetValue, \ + count, \ + gcvFALSE, gcvFALSE \ + ) + +#define _STATE_MIRROR_COUNT(reg, mirror, count) \ + _StateMirror(\ + Context, \ + reg ## _Address >> 2, \ + count, \ + mirror ## _Address >> 2 \ + ) + +#define _STATE_HINT(reg) \ + _State(\ + Context, index, \ + reg ## _Address >> 2, \ + reg ## _ResetValue, \ + reg ## _Count, \ + gcvFALSE, gcvTRUE \ + ) + +#define _STATE_HINT_BLOCK(reg, block, count) \ + _State(\ + Context, index, \ + (reg ## _Address >> 2) + (block << reg ## _BLK), \ + reg ## _ResetValue, \ + count, \ + gcvFALSE, gcvTRUE \ + ) + +#define _STATE_COUNT_OFFSET_HINT(reg, offset, count) \ + _State(\ + Context, index, \ + (reg ## _Address >> 2) + offset, \ + reg ## _ResetValue, \ + count, \ + gcvFALSE, gcvTRUE \ + ) + +#define _STATE_X(reg) \ + _State(\ + Context, index, \ + reg ## _Address >> 2, \ + reg ## _ResetValue, \ + reg ## _Count, \ + gcvTRUE, gcvFALSE \ + ) + +#define _STATE_INIT_VALUE(reg, value) \ + _State(\ + Context, index, \ + reg ## _Address >> 2, \ + value, \ + reg ## _Count, \ + gcvFALSE, gcvFALSE \ + ) + +#define _STATE_INIT_VALUE_BLOCK(reg, value, block, count) \ + _State(\ + Context, index, \ + (reg ## _Address >> 2) + (block << reg ## _BLK), \ + value, \ + count, \ + gcvFALSE, gcvFALSE \ + ) + + +#define _CLOSE_RANGE() \ + _TerminateStateBlock(Context, index) + +#define _ENABLE(reg, field) \ + do \ + { \ + if (gcmVERIFYFIELDVALUE(data, reg, MASK_ ## field, ENABLED)) \ + { \ + enable |= gcmFIELDMASK(reg, field); \ + } \ + } \ + while (gcvFALSE) + +#define _BLOCK_COUNT(reg) \ + ((reg ## _Count) >> (reg ## _BLK)) + + +/******************************************************************************\ +*********************** Support Functions and Definitions ********************** +\******************************************************************************/ + +#define gcdSTATE_MASK \ + (gcmSETFIELDVALUE(0, AQ_COMMAND_NOP_COMMAND, OPCODE, NOP) | 0xC0FFEE) + +#if gcdENABLE_3D +static gctUINT32 +_TerminateStateBlock( + IN gckCONTEXT Context, + IN gctUINT32 Index + ) +{ + gctUINT32_PTR buffer; + gctUINT32 align; + + /* Determine if we need alignment. */ + align = (Index & 1) ? 1 : 0; + + /* Address correct index. */ + buffer = (Context->buffer == gcvNULL) + ? gcvNULL + : Context->buffer->logical; + + /* Flush the current state block; make sure no pairing with the states + to follow happens. */ + if (align && (buffer != gcvNULL)) + { + buffer[Index] = 0xDEADDEAD; + } + + /* Reset last address. */ + Context->lastAddress = ~0U; + + /* Return alignment requirement. */ + return align; +} +#endif + + +#if gcdENABLE_3D +static gctUINT32 +_FlushPipe( + IN gckCONTEXT Context, + IN gctUINT32 Index, + IN gcePIPE_SELECT Pipe + ) +{ + gctUINT32 flushSlots; + gctBOOL txCacheFix; + gctBOOL fcFlushStall; + gctBOOL iCacheInvalidate; + gctBOOL halti5; + gctBOOL snapPages; + gctBOOL hwTFB; + gctBOOL blt; + gctBOOL peTSFlush; + gctBOOL multiCluster; + + txCacheFix + = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_TEX_CACHE_FLUSH_FIX); + + fcFlushStall + = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_FC_FLUSH_STALL); + + iCacheInvalidate + = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_SHADER_HAS_INSTRUCTION_CACHE); + + halti5 + = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_HALTI5); + + snapPages + = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_SNAPPAGE_CMD_FIX) && + gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_SNAPPAGE_CMD); + + hwTFB + = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_HW_TFB); + + blt + = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_BLT_ENGINE); + multiCluster + = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_MULTI_CLUSTER); + + peTSFlush + = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_PE_TILE_CACHE_FLUSH_FIX); + + flushSlots = blt ? 10 : 6; + + if (Pipe == gcvPIPE_3D) + { + if (!txCacheFix) + { + /* Semaphore stall */ + flushSlots += blt ? 8 : 4; + } + + /* VST cache */ + flushSlots += 2; + } + + if (fcFlushStall) + { + /* Flush tile status cache. */ + flushSlots += blt ? ((!peTSFlush) ? 14 :10) : 6; + } + + if (iCacheInvalidate && !halti5) + { + flushSlots += blt ? 16 : 12; + } + + if (hwTFB) + { + flushSlots += 2; + } + + /* Snap pages */ + if (snapPages) + { + flushSlots += 2; + } + + if (Context->buffer != gcvNULL) + { + gctUINT32_PTR buffer; + + /* Address correct index. */ + buffer = Context->buffer->logical + Index; + + if (Pipe == gcvPIPE_3D && !txCacheFix) + { + if (blt) + { + /* Semaphore from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + else + { + /* Semaphore from FE to PE. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to PE. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + } + + /* Flush the current pipe. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = (Pipe == gcvPIPE_2D) + ? + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))) + : ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) + | (multiCluster ? + 0 : ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2)))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 10:10) - (0 ? + 10:10) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 10:10) - (0 ? + 10:10) + 1))))))) << (0 ? + 10:10))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 10:10) - (0 ? + 10:10) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 10:10) - (0 ? 10:10) + 1))))))) << (0 ? 10:10))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 11:11) - (0 ? + 11:11) + 1))))))) << (0 ? + 11:11))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 11:11) - (0 ? 11:11) + 1))))))) << (0 ? 11:11))); + + if (hwTFB) + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x7003) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ + = 0x1; + } + + /* Flush VST in separate cmd. */ + if (Pipe == gcvPIPE_3D) + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))); + } + + /* Semaphore from FE to PE. */ + if (blt) + { + /* Semaphore from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + else + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to PE. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + + if (fcFlushStall) + { + if (!peTSFlush && blt) + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502B) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + else + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0594) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + + /* Semaphore from FE to PE. */ + if (blt) + { + /* Semaphore from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + else + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to PE. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + } + + if (iCacheInvalidate && !halti5) + { + /* Invalidate I$ after pipe is stalled */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0218) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x021A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0218) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x021A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))); + + /* Semaphore from FE to PE. */ + if (blt) + { + /* Semaphore from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + else + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to PE. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + } + + if (snapPages) + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x13 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x02 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x04 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x08 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))); + + *buffer++ + = 0; + } + } + + /* Number of slots taken by flushing pipe. */ + return flushSlots; +} +#endif + +#if gcdENABLE_3D +static gctUINT32 +_SemaphoreStall( + IN gckCONTEXT Context, + IN gctUINT32 Index + ) +{ + gctBOOL blt = gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_BLT_ENGINE); + if (Context->buffer != gcvNULL) + { + gctUINT32_PTR buffer; + + /* Address correct index. */ + buffer = Context->buffer->logical + Index; + + if (blt) + { + /* Semaphore from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to BLT. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + else + { + /* Semaphore from FE to PE. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to PE. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *buffer + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + } + + /* Semaphore/stall takes 4 slots. */ + return (blt ? 8 : 4); +} +#endif + +#if (gcdENABLE_3D) +static gctUINT32 +_SwitchPipe( + IN gckCONTEXT Context, + IN gctUINT32 Index, + IN gcePIPE_SELECT Pipe + ) +{ + gctUINT32 slots = 2; + + if (Context->buffer != gcvNULL) + { + gctUINT32_PTR buffer; + + /* Address correct index. */ + buffer = Context->buffer->logical + Index; + + /* LoadState(AQPipeSelect, 1), pipe. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E00) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer + = (Pipe == gcvPIPE_2D) + ? 0x1 + : 0x0; + } + + Context->pipeSelectBytes = slots * gcmSIZEOF(gctUINT32); + + return slots; +} +#endif + +#if gcdENABLE_3D +static gctUINT32 +_State( + IN gckCONTEXT Context, + IN gctUINT32 Index, + IN gctUINT32 Address, + IN gctUINT32 Value, + IN gctUINT32 Size, + IN gctBOOL FixedPoint, + IN gctBOOL Hinted + ) +{ + gctUINT32_PTR buffer; + gctUINT32 align; + gctUINT32 i; + + /* Determine if we need alignment. */ + align = (Index & 1) ? 1 : 0; + + /* Address correct index. */ + buffer = (Context->buffer == gcvNULL) + ? gcvNULL + : Context->buffer->logical; + + if ((buffer == gcvNULL) && (Address + Size > Context->maxState)) + { + /* Determine maximum state. */ + Context->maxState = Address + Size; + } + + if (buffer == gcvNULL) + { + /* Update number of states. */ + Context->numStates += Size; + } + + /* Do we need a new entry? */ + if ((Address != Context->lastAddress) || (FixedPoint != Context->lastFixed)) + { + if (buffer != gcvNULL) + { + if (align) + { + /* Add filler. */ + buffer[Index++] = 0xDEADDEAD; + } + + /* LoadState(Address, Count). */ + gcmkASSERT((Index & 1) == 0); + + if (FixedPoint) + { + buffer[Index] + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 26:26) - (0 ? + 26:26) + 1))))))) << (0 ? + 26:26))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 26:26) - (0 ? 26:26) + 1))))))) << (0 ? 26:26))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (Size) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (Address) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + } + else + { + buffer[Index] + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 26:26) - (0 ? + 26:26) + 1))))))) << (0 ? + 26:26))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 26:26) - (0 ? 26:26) + 1))))))) << (0 ? 26:26))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (Size) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (Address) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + } + + /* Walk all the states. */ + for (i = 0; i < (gctUINT32)Size; i += 1) + { + /* Set state to uninitialized value. */ + buffer[Index + 1 + i] = Value; + + /* Set index in state mapping table. */ + Context->map[Address + i].index = (gctUINT)Index + 1 + i; + } + } + + /* Save information for this LoadState. */ + Context->lastIndex = (gctUINT)Index; + Context->lastAddress = Address + (gctUINT32)Size; + Context->lastSize = Size; + Context->lastFixed = FixedPoint; + + /* Return size for load state. */ + return align + 1 + Size; + } + + /* Append this state to the previous one. */ + if (buffer != gcvNULL) + { + /* Update last load state. */ + buffer[Context->lastIndex] = + ((((gctUINT32) (buffer[Context->lastIndex])) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (Context->lastSize + Size) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + /* Walk all the states. */ + for (i = 0; i < (gctUINT32)Size; i += 1) + { + /* Set state to uninitialized value. */ + buffer[Index + i] = Value; + + /* Set index in state mapping table. */ + Context->map[Address + i].index = (gctUINT)Index + i; + } + } + + /* Update last address and size. */ + Context->lastAddress += (gctUINT32)Size; + Context->lastSize += Size; + + /* Return number of slots required. */ + return Size; +} + +static gctUINT32 +_StateMirror( + IN gckCONTEXT Context, + IN gctUINT32 Address, + IN gctUINT32 Size, + IN gctUINT32 AddressMirror + ) +{ + gctUINT32 i; + + /* Process when buffer is set. */ + if (Context->buffer != gcvNULL) + { + /* Walk all states. */ + for (i = 0; i < Size; i++) + { + /* Copy the mapping address. */ + Context->map[Address + i].index = + Context->map[AddressMirror + i].index; + } + } + + /* Return the number of required maps. */ + return Size; +} + +static void +_UpdateUnifiedReg( + IN gckCONTEXT Context, + IN gctUINT32 Address, + IN gctUINT32 Size, + IN gctUINT32 Count + ) +{ + gctUINT base; + gctUINT nopCount; + gctUINT32_PTR nop; + gcsCONTEXT_PTR buffer; + gcsSTATE_MAP_PTR map; + gctUINT i; + + /* Get the current context buffer. */ + buffer = Context->buffer; + + /* Get the state map. */ + map = Context->map; + + base = map[Address].index; + + if (Count > 1024) + { + buffer->logical[base - 1] + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 26:26) - (0 ? + 26:26) + 1))))))) << (0 ? + 26:26))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 26:26) - (0 ? 26:26) + 1))))))) << (0 ? 26:26))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1024) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (Address) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + buffer->logical[base + 1024 + 1] + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 26:26) - (0 ? + 26:26) + 1))))))) << (0 ? + 26:26))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 26:26) - (0 ? 26:26) + 1))))))) << (0 ? 26:26))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (Count - 1024) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (Address + 1024) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + /* Determine the number of NOP commands. */ + nopCount = (Size / 2) - (Count / 2); + /* Determine the location of the first NOP. */ + nop = &buffer->logical[base + (Count | 1) + 2]; + + /* Fill the unused space with NOPs. */ + for (i = 0; i < nopCount; i += 1) + { + if (nop >= buffer->logical + Context->totalSize) + { + break; + } + + /* Generate a NOP command. */ + *nop = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + /* Advance. */ + nop += 2; + } + } + else + { + buffer->logical[base - 1] + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 26:26) - (0 ? + 26:26) + 1))))))) << (0 ? + 26:26))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 26:26) - (0 ? 26:26) + 1))))))) << (0 ? 26:26))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (Count) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (Address) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + /* Determine the number of NOP commands. */ + nopCount = (Size / 2) - (Count / 2) + Size / 1024; + + /* Determine the location of the first NOP. */ + nop = &buffer->logical[base + (Count | 1)]; + + /* Fill the unused space with NOPs. */ + for (i = 0; i < nopCount; i += 1) + { + if (nop >= buffer->logical + Context->totalSize) + { + break; + } + + /* Generate a NOP command. */ + *nop = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + /* Advance. */ + nop += 2; + } + } +} +#endif + +#if (gcdENABLE_3D) +static gceSTATUS +_InitializeContextBuffer( + IN gckCONTEXT Context + ) +{ + gctUINT32_PTR buffer; + gctUINT32 index; + +#if gcdENABLE_3D + gctBOOL halti0, halti1, halti2, halti3, halti4, halti5; + gctUINT i; + gctUINT vertexUniforms, fragmentUniforms; + gctBOOL unifiedUniform; + gctBOOL hasGS, hasTS; + gctBOOL genericAttrib; + gctBOOL hasICache; + gctBOOL hasICachePrefetch; + gctUINT numRT = 0; + gctUINT numSamplers = 32; + gctBOOL hasTXdesc; + gctBOOL hasSecurity; + gctBOOL hasRobustness; + gctBOOL multiCluster; + gctBOOL smallBatch; + gctBOOL multiCoreBlockSetCfg2; + gctUINT clusterAliveMask; + gctBOOL hasPSCSThrottle; + gctBOOL hasMsaaFragOperation; + gctBOOL newGPipe; +#endif + + gckHARDWARE hardware; + + gcmkHEADER(); + + hardware = Context->hardware; + + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Reset the buffer index. */ + index = 0; + + /* Reset the last state address. */ + Context->lastAddress = ~0U; + + /* Get the buffer pointer. */ + buffer = (Context->buffer == gcvNULL) + ? gcvNULL + : Context->buffer->logical; + + + /**************************************************************************/ + /* Build 2D states. *******************************************************/ + + +#if gcdENABLE_3D + /**************************************************************************/ + /* Build 3D states. *******************************************************/ + + halti0 = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_HALTI0); + halti1 = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_HALTI1); + halti2 = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_HALTI2); + halti3 = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_HALTI3); + halti4 = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_HALTI4); + halti5 = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_HALTI5); + hasGS = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_GEOMETRY_SHADER); + hasTS = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_TESSELLATION); + genericAttrib = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_GENERIC_ATTRIB); + hasICache = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_SHADER_HAS_INSTRUCTION_CACHE); + hasTXdesc = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_TX_DESCRIPTOR); + hasSecurity = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_SECURITY); + hasRobustness = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_ROBUSTNESS); + hasICachePrefetch = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_SH_INSTRUCTION_PREFETCH); + multiCluster = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_MULTI_CLUSTER); + smallBatch = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_SMALL_BATCH) && hardware->options.smallBatch; + multiCoreBlockSetCfg2 = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_MULTI_CORE_BLOCK_SET_CONFIG2); + clusterAliveMask = hardware->identity.clusterAvailMask & hardware->options.userClusterMask; + hasPSCSThrottle = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_PSCS_THROTTLE); + hasMsaaFragOperation = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_MSAA_FRAGMENT_OPERATION); + newGPipe = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_NEW_GPIPE); + + /* Multi render target. */ + if (Context->hardware->identity.chipModel == gcv880 && + Context->hardware->identity.chipRevision == 0x5124 && + Context->hardware->identity.customerID == 0x103) + { + numRT = 16; + } + else if (halti2 || + ((Context->hardware->identity.chipModel == gcv900) && + (Context->hardware->identity.chipRevision == 0x5250))) + { + numRT = 8; + } + else if (halti0) + { + numRT = 4; + } + else + { + numRT = 1; + } + + if (hasGS && hasTS) + { + numSamplers = 80; + } + + /* Query how many uniforms can support. */ + gcmCONFIGUREUNIFORMS2(Context->hardware->identity.chipModel, + Context->hardware->identity.chipRevision, + halti5, + smallBatch, + Context->hardware->identity.numConstants, + unifiedUniform, + vertexUniforms, + fragmentUniforms); + +#if !gcdENABLE_UNIFIED_CONSTANT + if (Context->hardware->identity.numConstants > 256) + { + unifiedUniform = gcvTRUE; + } + else + { + unifiedUniform = gcvFALSE; + } +#endif + + /* Store the 3D entry index. */ + Context->entryOffset3D = (gctUINT)index * gcmSIZEOF(gctUINT32); + + /* Switch to 3D pipe. */ + index += _SwitchPipe(Context, index, gcvPIPE_3D); + + if (multiCluster) + { + index += _State(Context, index, (0x03910 >> 2) + (0 << 2), ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (clusterAliveMask) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))), 4, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x03908 >> 2, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:0) - (0 ? + 2:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:0) - (0 ? + 2:0) + 1))))))) << (0 ? + 2:0))) | (((gctUINT32) (0x2 & ((gctUINT32) ((((1 ? + 2:0) - (0 ? + 2:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:0) - (0 ? 2:0) + 1))))))) << (0 ? 2:0))), 1, gcvFALSE, gcvFALSE); + } + + /* Current context pointer. */ +#if gcdDEBUG + index += _State(Context, index, 0x03850 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); +#endif + + index += _FlushPipe(Context, index, gcvPIPE_3D); + + /* Global states. */ + if (hasSecurity) + { + index += _State(Context, index, 0x03900 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + index += _State(Context, index, 0x03904 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + index += _State(Context, index, 0x03814 >> 2, 0x00000001, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + index += _State(Context, index, 0x03818 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0381C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (halti5) + { + gctUINT32 uscControl = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 20:16) - (0 ? + 20:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 20:16) - (0 ? + 20:16) + 1))))))) << (0 ? + 20:16))) | (((gctUINT32) ((gctUINT32) (2) & ((gctUINT32) ((((1 ? + 20:16) - (0 ? + 20:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 20:16) - (0 ? 20:16) + 1))))))) << (0 ? 20:16))); + index += _State(Context, index, 0x03888 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x038C0 >> 2, 0x00000000, 16, gcvFALSE, gcvFALSE); + + uscControl |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:0) - (0 ? + 2:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:0) - (0 ? + 2:0) + 1))))))) << (0 ? + 2:0))) | (((gctUINT32) ((gctUINT32) (hardware->options.uscL1CacheRatio) & ((gctUINT32) ((((1 ? + 2:0) - (0 ? + 2:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:0) - (0 ? 2:0) + 1))))))) << (0 ? 2:0))); + if (multiCluster) + { + uscControl |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 11:8) - (0 ? + 11:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 11:8) - (0 ? + 11:8) + 1))))))) << (0 ? + 11:8))) | (((gctUINT32) ((gctUINT32) (hardware->options.uscAttribCacheRatio) & ((gctUINT32) ((((1 ? + 11:8) - (0 ? + 11:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 11:8) - (0 ? 11:8) + 1))))))) << (0 ? 11:8))); + } + index += _State(Context, index, 0x03884 >> 2, uscControl, 1, gcvFALSE, gcvFALSE); + } + else + { + index += _State(Context, index, 0x03820 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x03828 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0382C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x03834 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x03838 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x03854 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + if (hasGS) + { + index += _State(Context, index, 0x0388C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + index += _State(Context, index, 0x0384C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + /* Front End states. */ + if (halti5) + { + index += _State(Context, index, 0x17800 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + index += _State(Context, index, 0x007C4 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x007D0 >> 2, 0x00000000, 2, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x007D8 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x17A80 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + if (genericAttrib || newGPipe) + { + index += _State(Context, index, 0x17880 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x17900 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x17980 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x17A00 >> 2, 0x3F800000, 32, gcvFALSE, gcvFALSE); + } + } + else + { + index += _State(Context, index, 0x00600 >> 2, 0x00000000, (halti0 ? 16 : 12), gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + if (genericAttrib) + { + index += _State(Context, index, 0x006C0 >> 2, 0x00000000, 16, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00700 >> 2, 0x00000000, 16, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00740 >> 2, 0x00000000, 16, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00780 >> 2, 0x3F800000, 16, gcvFALSE, gcvFALSE); + } + } + + if (halti2 || (Context->hardware->identity.streamCount > 8)) + { + index += _State(Context, index, 0x14600 >> 2, 0x00000000, 16, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14640 >> 2, 0x00000000, 16, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14680 >> 2, 0x00000000, 16, gcvFALSE, gcvFALSE); + } + else if (Context->hardware->identity.streamCount > 1) + { + index += _State(Context, index, 0x00680 >> 2, 0x00000000, 8, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x006A0 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + } + else + { + index += _State(Context, index, 0x0064C >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x00650 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + } + index += _State(Context, index, 0x00644 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x00648 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00674 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (halti1) + { + index += _State(Context, index, 0x00678 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0067C >> 2, 0xFFFFFFFF, 1, gcvFALSE, gcvFALSE); + } + + index += _CLOSE_RANGE(); + + if (hasRobustness) + { + index += _State(Context, index, 0x146C0 >> 2, 0x00000000, 16, gcvFALSE, gcvTRUE); + index += _CLOSE_RANGE(); + index += _State(Context, index, 0x007F8 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _CLOSE_RANGE(); + } + + /* WD */ + if (multiCluster) + { + index += _State(Context, index, 0x18404 >> 2, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:0) - (0 ? + 1:0) + 1))))))) << (0 ? + 1:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:0) - (0 ? 1:0) + 1))))))) << (0 ? 1:0))), 1, gcvFALSE, gcvFALSE); + } + + if (halti5) + { + index += _State(Context, index, 0x008B8 >> 2, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) ((gctUINT32) (smallBatch ? + 0x0 : 0x1) & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) ((gctUINT32) (smallBatch ? + 0x0 : 0x1) & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:8) - (0 ? + 8:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:8) - (0 ? + 8:8) + 1))))))) << (0 ? + 8:8))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 8:8) - (0 ? + 8:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:8) - (0 ? + 8:8) + 1))))))) << (0 ? + 8:8))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))), 1, gcvFALSE, gcvFALSE); + + index += _State(Context, index, 0x15600 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + else + { + /* This register is programed by all chips, which program all DECODE_SELECT as VS + ** except SAMPLER_DECODE_SELECT. + */ + index += _State(Context, index, 0x00860 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + if (hasICache) + { + /* I-Cache states. */ + index += _State(Context, index, 0x00868 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0086C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0304C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01028 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + + if (hasICachePrefetch) + { + if (halti5) + { + index += _State(Context, index, 0x15604 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x01094 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + } + else + { + index += _State(Context, index, 0x00890 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x0104C >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + } + index += _CLOSE_RANGE(); + } + } + + if (multiCluster) + { + index += _State(Context, index, 0x010A8 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + /* Vertex Shader states. */ + index += _State(Context, index, 0x00804 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00808 >> 2, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:0) - (0 ? + 5:0) + 1))))))) << (0 ? + 5:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:0) - (0 ? 5:0) + 1))))))) << (0 ? 5:0))), 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0080C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00830 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (halti5) + { + index += _State(Context, index, 0x00898 >> 2, 0x00000000, 2, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x008A0 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00870 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x008A8 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x008C0 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x008E0 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + } + else + { + index += _State(Context, index, 0x00810 >> 2, 0x00000000, 4, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00820 >> 2, 0x00000000, 4, gcvFALSE, gcvFALSE); + } + + if (multiCluster) + { + index += _State(Context, index, 0x007FC >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x15608 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + index += _CLOSE_RANGE(); + + /* GS */ + if (hasGS) + { + index += _State(Context, index, 0x01100 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01104 >> 2, 0x00000001, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01108 >> 2, 0x01000001, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0110C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01110 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01114 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x0111C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01140 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01144 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01148 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0114C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01154 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01120 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + } + + /* TCS & TES */ + if (hasTS) + { + index += _State(Context, index, 0x007C0 >> 2, 0x00000003, 1, gcvFALSE, gcvFALSE); + + index += _State(Context, index, 0x14A14 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14A18 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14A1C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14A40 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14A00 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14A04 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14A08 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x14A10 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14A20 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14A44 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14A4C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + index += _CLOSE_RANGE(); + + index += _State(Context, index, 0x14B18 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14B1C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14B20 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14B04 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14B08 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14B0C >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x14B14 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14B40 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14B24 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14B2C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14B34 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + index += _State(Context, index, 0x14B00 >> 2, 0x00040000, 1, gcvFALSE, gcvFALSE); + + } + + index += _CLOSE_RANGE(); + + /* TFB */ + if (gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_HW_TFB)) + { + index += _State(Context, index, 0x1C000 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x1C008 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x1C040 >> 2) + (0 << 4), 0x00000000, 4, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x1C080 >> 2, 0x00000000, 4, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x1C0C0 >> 2, 0x00000000, 4, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x1C100 >> 2, 0x00000000, 4, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x1C800 >> 2, 0x00000000, 128*4, gcvFALSE, gcvFALSE); + + index += _State(Context, index, 0x1C014 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _CLOSE_RANGE(); + + } + + /* Primitive Assembly states. */ + index += _State(Context, index, 0x00A00 >> 2, 0x00000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00A04 >> 2, 0x00000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00A08 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A0C >> 2, 0x00000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00A10 >> 2, 0x00000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00A14 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A18 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A1C >> 2, 0x3F000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A28 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A2C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A30 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A34 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A38 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A3C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A80 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A84 >> 2, 0x00000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00A8C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A88 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (halti5) + { + index += _State(Context, index, 0x00AA8 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00A90 >> 2, 0x00000000, 4, gcvFALSE, gcvFALSE); + } + else + { + index += _State(Context, index, 0x00A40 >> 2, 0x00000000, Context->hardware->identity.varyingsCount, gcvFALSE, gcvFALSE); + } + + if (multiCluster) + { + index += _State(Context, index, 0x00AAC >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + index += _State(Context, index, 0x03A00 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x03A04 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x03A08 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (multiCoreBlockSetCfg2) + { + index += _State(Context, index, 0x03A0C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x03A10 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + /* Setup states. */ + index += _State(Context, index, 0x00C00 >> 2, 0x00000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00C04 >> 2, 0x00000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00C08 >> 2, 0x45000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00C0C >> 2, 0x45000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00C10 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00C14 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00C18 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00C1C >> 2, 0x42000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00C20 >> 2, 0x00000000, 1, gcvTRUE, gcvFALSE); + index += _State(Context, index, 0x00C24 >> 2, 0x00000000, 1, gcvTRUE, gcvFALSE); + + /* Raster states. */ + index += _State(Context, index, 0x00E00 >> 2, 0x000000F1, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00E10 >> 2, 0x00000000, 4, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00E04 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00E40 >> 2, 0x00000000, 16, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00E08 >> 2, ((((gctUINT32) (0x17000031)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) ((gctUINT32) (smallBatch ? + 0x0 : 0x1) & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2))), 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00E24 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00E20 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (halti2) + { + index += _State(Context, index, 0x00E0C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + if (halti5) + { + index += _State(Context, index, 0x00E34 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + + /* Pixel Shader states. */ + index += _State(Context, index, 0x01004 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01008 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0100C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01010 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01030 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01034 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (halti2) + { + index += _State(Context, index, 0x01040 >> 2, 0x00000000, 2, gcvFALSE, gcvFALSE); + } + + if (numRT == 16) + { + index += _State(Context, index, 0x0102C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x010C8 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x010CC >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + else if (numRT == 8) + { + index += _State(Context, index, 0x0102C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01038 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + if (hasMsaaFragOperation) + { + index += _State(Context, index, 0x01054 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01060 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + } + + if (halti5) + { + index += _State(Context, index, 0x01080 >> 2, 0x00000000, 4, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01058 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01098 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + if (hasPSCSThrottle) + { + index += _State(Context, index, 0x0109C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + index += _CLOSE_RANGE(); + + /* Texture states. */ + if (hasTXdesc) + { + /* Texture descriptor states */ + index += _State(Context, index, 0x14C40 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (smallBatch) + { + index += _State(Context, index, 0x010B0 >> 2, numSamplers, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x010B4 >> 2, numSamplers, 1, gcvFALSE, gcvFALSE); + + index += _State(Context, index, 0x16000 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x16200 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x16400 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x16600 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x16800 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + + index += _State(Context, index, (0x15800 >> 2) + (0 << 0), 0x00000000, numSamplers, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x15A00 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + + index += _CLOSE_RANGE(); + } + else + { + index += _State(Context, index, 0x16C00 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x16E00 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x17000 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x17200 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x17400 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + + index += _State(Context, index, (0x15C00 >> 2) + (0 << 0), 0x00000000, numSamplers, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x15E00 >> 2, 0x00000000, numSamplers, gcvFALSE, gcvFALSE); + + index += _CLOSE_RANGE(); + + _StateMirror(Context, 0x16000 >> 2, numSamplers , 0x16C00 >> 2); + _StateMirror(Context, 0x16200 >> 2, numSamplers , 0x16E00 >> 2); + _StateMirror(Context, 0x16400 >> 2, numSamplers , 0x17000 >> 2); + _StateMirror(Context, 0x16600 >> 2, numSamplers , 0x17200 >> 2); + _StateMirror(Context, 0x16800 >> 2, numSamplers , 0x17400 >> 2); + _StateMirror(Context, 0x15800 >> 2, numSamplers , 0x15C00 >> 2); + _StateMirror(Context, 0x15A00 >> 2, numSamplers , 0x15E00 >> 2); + } + } + else + { + index += _State(Context, index, 0x02000 >> 2, 0x00000000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x02040 >> 2, 0x00000000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x02080 >> 2, 0x00000000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x020C0 >> 2, 0x00000000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x02100 >> 2, 0x00000000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x02140 >> 2, 0x00000000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x02180 >> 2, 0x00000000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x021C0 >> 2, 0x00321000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x02200 >> 2, 0x00000000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x02240 >> 2, 0x00000000, 12, gcvFALSE, gcvFALSE); + index += _State(Context, index, (0x02400 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02440 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02480 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x024C0 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02500 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02540 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02580 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x025C0 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02600 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02640 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02680 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x026C0 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02700 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x02740 >> 2) + (0 << 4), 0x00000000, 12, gcvFALSE, gcvTRUE); + index += _CLOSE_RANGE(); + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_TEXTURE_LINEAR)) + { + /* + * Linear stride LODn will overwrite LOD0 on GC880,GC2000. + * And only LOD0 is valid for this register. + */ + gctUINT count = halti1 ? 14 : 1; + + for (i = 0; i < 12; i += 1) + { + index += _State(Context, index, (0x02C00 >> 2) + i * 16, 0x00000000, count, gcvFALSE, gcvFALSE); + } + } + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_SUPPORT_GCREGTX)) + { + gctUINT texBlockCount; + gctUINT gcregTXLogSizeResetValue; + + /* Enable the integer filter pipe for all texture samplers + so that the floating point filter clock will shut off until + we start using the floating point filter. + */ + gcregTXLogSizeResetValue = ((((gctUINT32) (0x00000000)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 29:29) - (0 ? + 29:29) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 29:29) - (0 ? + 29:29) + 1))))))) << (0 ? + 29:29))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 29:29) - (0 ? + 29:29) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 29:29) - (0 ? 29:29) + 1))))))) << (0 ? 29:29))); + + /* New texture block. */ + index += _State(Context, index, 0x10000 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10080 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10100 >> 2, gcregTXLogSizeResetValue, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10180 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10200 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10280 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10300 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10380 >> 2, 0x00321000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10400 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10480 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_TX_FILTER)) + { + index += _State(Context, index, 0x12000 >> 2, 0x00000000, 256, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x12400 >> 2, 0x00000000, 256, gcvFALSE, gcvFALSE); + } + + texBlockCount = ((512) >> (4)); + + for (i = 0; i < texBlockCount; i += 1) + { + index += _State(Context, index, (0x10800 >> 2) + (i << 4), 0x00000000, 14, gcvFALSE, gcvTRUE); + } + } + + if (gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_TEX_BASELOD)) + { + index += _State(Context, index, 0x10700 >> 2, 0x00000F00, 32, gcvFALSE, gcvFALSE); + } + + if (halti3 || + gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_TX_SUPPORT_DEC)) + { + index += _State(Context, index, 0x10780 >> 2, 0x00030000, 32, gcvFALSE, gcvFALSE); + } + + if (halti4) + { + index += _State(Context, index, 0x11200 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x11280 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + } + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_TX_FRAC_PRECISION_6BIT)) + { + index += _State(Context, index, 0x11000 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x11080 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x11100 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x11180 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x11300 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + } + + /* ASTC */ + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_TEXTURE_ASTC)) + { + index += _State(Context, index, 0x10500 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10580 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10600 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x10680 >> 2, 0x00000000, 32, gcvFALSE, gcvFALSE); + } + } + + if (halti3) + { + index += _State(Context, index, 0x14C00 >> 2, 0x00000000, 16, gcvFALSE, gcvFALSE); + } + + /* Thread walker states. */ + index += _State(Context, index, 0x00900 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00904 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00908 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0090C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00910 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00914 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00918 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00924 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0091C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_SHADER_ENHANCEMENTS2)) + { + index += _State(Context, index, 0x00940 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00944 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00948 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0094C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00950 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00954 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + if (halti5) + { + index += _State(Context, index, 0x00958 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0095C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00960 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + index += _CLOSE_RANGE(); + + /* VS/PS Start/End PC register */ + if (halti5) + { + index += _State(Context, index, 0x00874 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x008BC >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0087C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01090 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + } + else if (hasICache) + { + /* New Shader instruction PC registers(20bit). */ + index += _State(Context, index, 0x00874 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00878 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0087C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00880 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + } + else + { + if (Context->hardware->identity.instructionCount <= 256) + { + /* old shader instruction PC registers (12bit)*/ + index += _State(Context, index, 0x00800 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00838 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + + index += _State(Context, index, 0x01000 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01018 >> 2, 0x01000000, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + } + else + { + /* New Shader instruction PC registers (16bit) */ + index += _State(Context, index, 0x0085C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0101C >> 2, 0x00000100, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + } + } + + + if (!hasICachePrefetch) + { + /* This unified one need SELECT bit to steer */ + if (Context->hardware->identity.instructionCount > 1024) + { + for (i = 0; + i < Context->hardware->identity.instructionCount << 2; + i += 256 << 2 + ) + { + index += _State(Context, index, (0x20000 >> 2) + i, 0x00000000, 256 << 2, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + } + } + /* This unified one is steered by base adddress, it's automatical. */ + else if (Context->hardware->identity.instructionCount > 256) + { + /* VS instruction memory. */ + for (i = 0; + i < Context->hardware->identity.instructionCount << 2; + i += 256 << 2 + ) + { + index += _State(Context, index, (0x0C000 >> 2) + i, 0x00000000, 256 << 2, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + } + + _StateMirror(Context, 0x08000 >> 2, Context->hardware->identity.instructionCount << 2 , 0x0C000 >> 2); + } + /* if (Context->hardware->identity.instructionCount <= 256). This is non-unified one. */ + else + { + index += _State(Context, index, 0x04000 >> 2, 0x00000000, 1024, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + index += _State(Context, index, 0x06000 >> 2, 0x00000000, 1024, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + } + } + + if (unifiedUniform) + { + gctINT numConstants = Context->hardware->identity.numConstants; + + /* Base Offset register */ + index += _State(Context, index, 0x01024 >> 2, 0x00000100, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x00864 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + + if (smallBatch) + { + index += _State(Context, index, 0x010AC >> 2, numConstants, 1, gcvFALSE, gcvFALSE); + } + + for (i = 0; + numConstants > 0; + i += 256 << 2, + numConstants -= 256 + ) + { + if (halti5) + { + if (numConstants >= 256) + { + if (smallBatch) + { + index += _State(Context, index, (0x34000 >> 2) + i, 0x00000000, 256 << 2, gcvFALSE, gcvFALSE); + } + else + { + index += _State(Context, index, (0x36000 >> 2) + i, 0x00000000, 256 << 2, gcvFALSE, gcvFALSE); + } + } + else + { + if (smallBatch) + { + index += _State(Context, index, (0x34000 >> 2) + i, 0x00000000, numConstants << 2, gcvFALSE, gcvFALSE); + } + else + { + index += _State(Context, index, (0x36000 >> 2) + i, 0x00000000, numConstants << 2, gcvFALSE, gcvFALSE); + } + } + index += _CLOSE_RANGE(); + } + else + { + if (numConstants >= 256) + { + index += _State(Context, index, (0x30000 >> 2) + i, 0x00000000, 256 << 2, gcvFALSE, gcvFALSE); + } + else + { + index += _State(Context, index, (0x30000 >> 2) + i, 0x00000000, numConstants << 2, gcvFALSE, gcvFALSE); + } + + index += _CLOSE_RANGE(); + } + } + + if (halti5 && !smallBatch) + { + _StateMirror(Context, 0x34000 >> 2, Context->hardware->identity.numConstants << 2 , 0x36000 >> 2); + } + } +#if gcdENABLE_UNIFIED_CONSTANT + else +#endif + { + index += _State(Context, index, 0x05000 >> 2, 0x00000000, vertexUniforms * 4, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x07000 >> 2, 0x00000000, fragmentUniforms * 4, gcvFALSE, gcvFALSE); + } + + if (halti1) + { + index += _State(Context, index, 0x00884 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + if (halti5) + { + index += _State(Context, index, 0x008B0 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + /* Store the index of the "XD" entry. */ + Context->entryOffsetXDFrom3D = (gctUINT)index * gcmSIZEOF(gctUINT32); + + + /* Pixel Engine states. */ + index += _State(Context, index, 0x01400 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01404 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01408 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0140C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01414 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01418 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0141C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01420 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01424 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01428 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x0142C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01434 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01454 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01458 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x014A0 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x014A8 >> 2, 0xFFFFFFFF, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x014AC >> 2, 0xFFFFFFFF, 1, gcvFALSE, gcvFALSE); + + if(gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_HALF_FLOAT_PIPE) ) + { + index += _State(Context, index, 0x014B0 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x014B4 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + index += _State(Context, index, 0x014A4 >> 2, 0x000E400C, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01580 >> 2, 0x00000000, 3, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x014B8 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + + if (halti3) + { + index += _State(Context, index, 0x0103C >> 2, 0x76543210, 1, gcvFALSE, gcvFALSE); + } + + index += _State(Context, index, (0x01460 >> 2) + (0 << 3), 0x00000000, Context->hardware->identity.pixelPipes, gcvFALSE, gcvTRUE); + + if (Context->hardware->identity.pixelPipes == 1) + { + index += _State(Context, index, 0x01430 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x01410 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + } + + if (Context->hardware->identity.pixelPipes > 1 || halti0) + { + index += _State(Context, index, (0x01480 >> 2) + (0 << 3), 0x00000000, Context->hardware->identity.pixelPipes, gcvFALSE, gcvTRUE); + } + + for (i = 0; i < 3; i++) + { + index += _State(Context, index, (0x01500 >> 2) + (i << 3), 0x00000000, Context->hardware->identity.pixelPipes, gcvFALSE, gcvTRUE); + } + + if (numRT == 16) + { + for (i = 0; i < 15; i++) + { + index += _State(Context, index, (0x17C00 >> 2) + (i << 0), 0x00000000, Context->hardware->identity.pixelPipes, gcvFALSE, gcvTRUE); + } + index += _State(Context, index, 0x17C40 >> 2, 0x00000000, 15, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x17C80 >> 2, 0x03012000, 15, gcvFALSE, gcvFALSE); + } + else if (numRT == 8) + { + for (i = 0; i < 7; i++) + { + index += _State(Context, index, (0x14800 >> 2) + (i << 3), 0x00000000, Context->hardware->identity.pixelPipes, gcvFALSE, gcvTRUE); + } + index += _State(Context, index, 0x14900 >> 2, 0x00000000, 7, gcvFALSE, gcvFALSE); + } + + + if (halti3) + { + index += _State(Context, index, 0x014BC >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + if (halti4) + { + index += _State(Context, index, 0x014C0 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + } + + if (hasGS) + { + index += _State(Context, index, 0x038A0 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + } + + if (halti5) + { + index += _State(Context, index, 0x14920 >> 2, 0x00000000, 7, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14940 >> 2, 0x00000000, 7, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14960 >> 2, 0x00000000, 7, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x14980 >> 2, 0x00000000, 7, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x149A0 >> 2, 0x00000000, 7, gcvFALSE, gcvFALSE); + } + + if (hasRobustness) + { + index += _State(Context, index, 0x149C0 >> 2, 0x00000000, 8, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x014C4 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + } + + /* Memory Controller */ + index += _State(Context, index, 0x01654 >> 2, 0x00200000, 1, gcvFALSE, gcvFALSE); + + index += _CLOSE_RANGE(); + index += _State(Context, index, 0x01658 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x0165C >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x01660 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01664 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x01668 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x0166C >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01670 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01674 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x016A4 >> 2, 0x00000000, 1, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x016AC >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x016A8 >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01720 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x01740 >> 2, 0x00000000, 8, gcvFALSE, gcvTRUE); + index += _State(Context, index, 0x01760 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + + + if (halti2) + { + index += _State(Context, index, 0x01780 >> 2, 0x00000000, 8, gcvFALSE, gcvFALSE); + index += _State(Context, index, 0x016BC >> 2, 0x00000000, 1, gcvFALSE, gcvFALSE); + index += _State(Context, index, (0x017A0 >> 2) + 1, 0x00000000, 7, gcvFALSE, gcvFALSE); + index += _State(Context, index, (0x017C0 >> 2) + 1, 0x00000000, 7, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x017E0 >> 2) + 1, 0x00000000, 7, gcvFALSE, gcvTRUE); + index += _State(Context, index, (0x01A00 >> 2) + 1, 0x00000000, 7, gcvFALSE, gcvFALSE); + index += _State(Context, index, (0x01A20 >> 2) + 1, 0x00000000, 7, gcvFALSE, gcvFALSE); + index += _State(Context, index, (0x01A40 >> 2) + 1, 0x00000000, 7, gcvFALSE, gcvFALSE); + } + + index += _CLOSE_RANGE(); + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_BUG_FIXES18)) + { + index += _State(Context, index, 0x03860 >> 2, 0x6, 1, gcvFALSE, gcvFALSE); + index += _CLOSE_RANGE(); + } + + if (halti3) + { + index += _State(Context, index, 0x01A80 >> 2, 0x00000000, 8, gcvFALSE, gcvTRUE); + index += _CLOSE_RANGE(); + } + + if (hasSecurity || hasRobustness) + { + index += _State(Context, index, 0x001AC >> 2, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))), 1, gcvFALSE, gcvFALSE); + } + + /* Semaphore/stall. */ + index += _SemaphoreStall(Context, index); +#endif + + /**************************************************************************/ + /* Link to another address. ***********************************************/ + + Context->linkIndex3D = (gctUINT)index; + + if (buffer != gcvNULL) + { + buffer[index + 0] + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x08 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + buffer[index + 1] + = 0; + } + + index += 2; + + /* Store the end of the context buffer. */ + Context->bufferSize = index * gcmSIZEOF(gctUINT32); + + + /**************************************************************************/ + /* Pipe switch for the case where neither 2D nor 3D are used. *************/ + + /* Store the 3D entry index. */ + Context->entryOffsetXDFrom2D = (gctUINT)index * gcmSIZEOF(gctUINT32); + + /* Switch to 3D pipe. */ + index += _SwitchPipe(Context, index, gcvPIPE_3D); + + /* Store the location of the link. */ + Context->linkIndexXD = (gctUINT)index; + + if (buffer != gcvNULL) + { + buffer[index + 0] + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x08 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + buffer[index + 1] + = 0; + } + + index += 2; + + + /**************************************************************************/ + /* Save size for buffer. **************************************************/ + + Context->totalSize = index * gcmSIZEOF(gctUINT32); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} +#endif + +static gceSTATUS +_DestroyContext( + IN gckCONTEXT Context + ) +{ + gceSTATUS status = gcvSTATUS_OK; + + if (Context != gcvNULL) + { + gcsCONTEXT_PTR bufferHead; + + /* Free context buffers. */ + for (bufferHead = Context->buffer; Context->buffer != gcvNULL;) + { + /* Get a shortcut to the current buffer. */ + gcsCONTEXT_PTR buffer = Context->buffer; + + /* Get the next buffer. */ + gcsCONTEXT_PTR next = buffer->next; + + /* Last item? */ + if (next == bufferHead) + { + next = gcvNULL; + } + + /* Destroy the signal. */ + if (buffer->signal != gcvNULL) + { + gcmkONERROR(gckOS_DestroySignal( + Context->os, buffer->signal + )); + + buffer->signal = gcvNULL; + } + + /* Free state delta map. */ + if (buffer->logical != gcvNULL) + { + gckKERNEL kernel = Context->hardware->kernel; + +#if gcdCAPTURE_ONLY_MODE + gceDATABASE_TYPE dbType; + gctUINT32 processID; +#endif + + /* End cpu access. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + kernel, + buffer->videoMem, + 0, + gcvFALSE, + gcvFALSE + )); + + /* Synchronized unlock. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( + kernel, + buffer->videoMem, + 0, + gcvNULL + )); + +#if gcdCAPTURE_ONLY_MODE + /* Encode surface type and pool to database type. */ + dbType = gcvDB_VIDEO_MEMORY + | (gcvVIDMEM_TYPE_GENERIC << gcdDB_VIDEO_MEMORY_TYPE_SHIFT) + | (buffer->videoMem->pool << gcdDB_VIDEO_MEMORY_POOL_SHIFT); + + gcmkONERROR(gckOS_GetProcessID(&processID)); + + gcmkONERROR( + gckKERNEL_RemoveProcessDB(kernel, + processID, + dbType, + buffer->videoMem)); +#endif + + /* Free video memory. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + kernel, + buffer->videoMem + )); + + buffer->logical = gcvNULL; + } + + /* Free context buffer. */ + gcmkONERROR(gcmkOS_SAFE_FREE(Context->os, buffer)); + + /* Remove from the list. */ + Context->buffer = next; + } + + /* Mark the gckCONTEXT object as unknown. */ + Context->object.type = gcvOBJ_UNKNOWN; + + /* Free the gckCONTEXT object. */ + gcmkONERROR(gcmkOS_SAFE_FREE(Context->os, Context)); + } + +OnError: + return status; +} + +#if (gcdENABLE_3D) +static gceSTATUS +_AllocateContextBuffer( + IN gckCONTEXT Context, + IN gcsCONTEXT_PTR Buffer + ) +{ + gceSTATUS status; + gckKERNEL kernel = Context->hardware->kernel; + gcePOOL pool = gcvPOOL_DEFAULT; + gctSIZE_T totalSize = Context->totalSize; + gctUINT32 allocFlag = 0; + +#if gcdCAPTURE_ONLY_MODE + gceDATABASE_TYPE dbType; + gctUINT32 processID; +#endif + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag = gcvALLOC_FLAG_CACHEABLE; +#endif + + /* Allocate video memory node for command buffers. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + allocFlag, + &totalSize, + &pool, + &Buffer->videoMem + )); + +#if gcdCAPTURE_ONLY_MODE + gcmkONERROR(gckVIDMEM_HANDLE_Allocate(kernel, Buffer->videoMem, &Context->buffer->handle)); + + /* Encode surface type and pool to database type. */ + dbType = gcvDB_VIDEO_MEMORY + | (gcvVIDMEM_TYPE_GENERIC << gcdDB_VIDEO_MEMORY_TYPE_SHIFT) + | (pool << gcdDB_VIDEO_MEMORY_POOL_SHIFT); + + gcmkONERROR(gckOS_GetProcessID(&processID)); + + /* Record in process db. */ + gcmkONERROR( + gckKERNEL_AddProcessDB(kernel, + processID, + dbType, + Buffer->videoMem, + gcvNULL, + totalSize)); +#endif + + /* Lock for GPU access. */ + gcmkONERROR(gckVIDMEM_NODE_Lock( + kernel, + Buffer->videoMem, + &Buffer->address + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + kernel, + Buffer->videoMem, + gcvFALSE, + gcvFALSE, + (gctPOINTER *)&Buffer->logical + )); + + return gcvSTATUS_OK; + +OnError: + return status; +} +#endif + +/******************************************************************************\ +**************************** Context Management API **************************** +\******************************************************************************/ + +/******************************************************************************\ +** +** gckCONTEXT_Construct +** +** Construct a new gckCONTEXT object. +** +** INPUT: +** +** gckOS Os +** Pointer to gckOS object. +** +** gctUINT32 ProcessID +** Current process ID. +** +** gckHARDWARE Hardware +** Pointer to gckHARDWARE object. +** +** OUTPUT: +** +** gckCONTEXT * Context +** Pointer to a variable thet will receive the gckCONTEXT object +** pointer. +*/ +#if (gcdENABLE_3D) +gceSTATUS +gckCONTEXT_Construct( + IN gckOS Os, + IN gckHARDWARE Hardware, + IN gctUINT32 ProcessID, + OUT gckCONTEXT * Context + ) +{ + gceSTATUS status; + gckCONTEXT context = gcvNULL; + gctUINT32 allocationSize; + gctUINT i; + gctPOINTER pointer = gcvNULL; + + gcmkHEADER_ARG("Os=%p Hardware=%p", Os, Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Os, gcvOBJ_OS); + gcmkVERIFY_ARGUMENT(Context != gcvNULL); + + + /**************************************************************************/ + /* Allocate and initialize basic fields of gckCONTEXT. ********************/ + + /* The context object size. */ + allocationSize = gcmSIZEOF(struct _gckCONTEXT); + + /* Allocate the object. */ + gcmkONERROR(gckOS_Allocate( + Os, allocationSize, &pointer + )); + + context = pointer; + + /* Reset the entire object. */ + gcmkONERROR(gckOS_ZeroMemory(context, allocationSize)); + + /* Initialize the gckCONTEXT object. */ + context->object.type = gcvOBJ_CONTEXT; + context->os = Os; + context->hardware = Hardware; + + +#if !gcdENABLE_3D + context->entryPipe = gcvPIPE_2D; + context->exitPipe = gcvPIPE_2D; +#elif gcdCMD_NO_2D_CONTEXT + context->entryPipe = gcvPIPE_3D; + context->exitPipe = gcvPIPE_3D; +#else + context->entryPipe + = (((((gctUINT32) (context->hardware->identity.chipFeatures)) >> (0 ? 9:9)) & ((gctUINT32) ((((1 ? 9:9) - (0 ? 9:9) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1)))))) ) + ? gcvPIPE_2D + : gcvPIPE_3D; + context->exitPipe = gcvPIPE_3D; +#endif + + /* Get the command buffer requirements. */ + gcmkONERROR(gckHARDWARE_QueryCommandBuffer( + Hardware, + gcvENGINE_RENDER, + &context->alignment, + &context->reservedHead, + gcvNULL + )); + + /**************************************************************************/ + /* Get the size of the context buffer. ************************************/ + + gcmkONERROR(_InitializeContextBuffer(context)); + + if (context->maxState > 0) + { + /**************************************************************************/ + /* Allocate and reset the state mapping table. ****************************/ + if (context->hardware->kernel->command->stateMap == gcvNULL) + { + /* Allocate the state mapping table. */ + gcmkONERROR(gckOS_Allocate( + Os, + gcmSIZEOF(gcsSTATE_MAP) * context->maxState, + &pointer + )); + + context->map = pointer; + + /* Zero the state mapping table. */ + gcmkONERROR(gckOS_ZeroMemory( + context->map, gcmSIZEOF(gcsSTATE_MAP) * context->maxState + )); + + context->hardware->kernel->command->stateMap = pointer; + } + else + { + context->map = context->hardware->kernel->command->stateMap; + } + } + + /**************************************************************************/ + /* Allocate the context and state delta buffers. **************************/ + + for (i = 0; i < gcdCONTEXT_BUFFER_COUNT; i += 1) + { + /* Allocate a context buffer. */ + gcsCONTEXT_PTR buffer; + + /* Allocate the context buffer structure. */ + gcmkONERROR(gckOS_Allocate( + Os, + gcmSIZEOF(gcsCONTEXT), + &pointer + )); + + buffer = pointer; + + /* Reset the context buffer structure. */ + gcmkVERIFY_OK(gckOS_ZeroMemory( + buffer, gcmSIZEOF(gcsCONTEXT) + )); + + /* Append to the list. */ + if (context->buffer == gcvNULL) + { + buffer->next = buffer; + context->buffer = buffer; + } + else + { + buffer->next = context->buffer->next; + context->buffer->next = buffer; + } + + /* Set the number of delta in the order of creation. */ +#if gcmIS_DEBUG(gcdDEBUG_CODE) + buffer->num = i; +#endif + + /* Create the busy signal. */ + gcmkONERROR(gckOS_CreateSignal( + Os, gcvFALSE, &buffer->signal + )); + + /* Set the signal, buffer is currently not busy. */ + gcmkONERROR(gckOS_Signal( + Os, buffer->signal, gcvTRUE + )); + + /* Create a new physical context buffer. */ + gcmkONERROR(_AllocateContextBuffer( + context, buffer + )); + + /* Set gckEVENT object pointer. */ + buffer->eventObj = Hardware->kernel->eventObj; + + /* Set the pointers to the LINK commands. */ + if (context->linkIndex2D != 0) + { + buffer->link2D = &buffer->logical[context->linkIndex2D]; + } + + if (context->linkIndex3D != 0) + { + buffer->link3D = &buffer->logical[context->linkIndex3D]; + } + + if (context->linkIndexXD != 0) + { + gctPOINTER xdLink; + gctUINT32 xdEntryAddress; + gctUINT32 xdEntrySize; + gctUINT32 linkBytes; + + /* Determine LINK parameters. */ + xdLink + = &buffer->logical[context->linkIndexXD]; + + xdEntryAddress + = buffer->address + + context->entryOffsetXDFrom3D; + + xdEntrySize + = context->bufferSize + - context->entryOffsetXDFrom3D; + + /* Query LINK size. */ + gcmkONERROR(gckWLFE_Link( + Hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL + )); + + /* Generate a LINK. */ + gcmkONERROR(gckWLFE_Link( + Hardware, + xdLink, + xdEntryAddress, + xdEntrySize, + &linkBytes, + gcvNULL, + gcvNULL + )); + } + } + + + /**************************************************************************/ + /* Initialize the context buffers. ****************************************/ + + /* Initialize the current context buffer. */ + gcmkONERROR(_InitializeContextBuffer(context)); + + /* Make all created contexts equal. */ + { + gcsCONTEXT_PTR currContext, tempContext; + + /* Set the current context buffer. */ + currContext = context->buffer; + + /* Get the next context buffer. */ + tempContext = currContext->next; + + /* Loop through all buffers. */ + while (tempContext != currContext) + { + if (tempContext == gcvNULL) + { + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + + /* Copy the current context. */ + gckOS_MemCopy( + tempContext->logical, + currContext->logical, + context->totalSize + ); + + /* Get the next context buffer. */ + tempContext = tempContext->next; + } + } + + /* Return pointer to the gckCONTEXT object. */ + *Context = context; + + /* Success. */ + gcmkFOOTER_ARG("*Context=0x%08X", *Context); + return gcvSTATUS_OK; + +OnError: + /* Roll back on error. */ + gcmkVERIFY_OK(_DestroyContext(context)); + + /* Return the status. */ + gcmkFOOTER(); + return status; +} +#endif + +/******************************************************************************\ +** +** gckCONTEXT_Destroy +** +** Destroy a gckCONTEXT object. +** +** INPUT: +** +** gckCONTEXT Context +** Pointer to an gckCONTEXT object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCONTEXT_Destroy( + IN gckCONTEXT Context + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Context=%p", Context); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Context, gcvOBJ_CONTEXT); + + /* Destroy the context and all related objects. */ + status = _DestroyContext(Context); + + /* Success. */ + gcmkFOOTER_NO(); + return status; +} + +/******************************************************************************\ +** +** gckCONTEXT_Update +** +** Merge all pending state delta buffers into the current context buffer. +** +** INPUT: +** +** gckCONTEXT Context +** Pointer to an gckCONTEXT object. +** +** gctUINT32 ProcessID +** Current process ID. +** +** gcsSTATE_DELTA_PTR StateDelta +** Pointer to the state delta. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCONTEXT_Update( + IN gckCONTEXT Context, + IN gctUINT32 ProcessID, + IN gcsSTATE_DELTA_PTR StateDelta + ) +{ +#if gcdENABLE_3D + gceSTATUS status = gcvSTATUS_OK; + gcsSTATE_DELTA _stateDelta; + gckKERNEL kernel; + gcsCONTEXT_PTR buffer; + gcsSTATE_MAP_PTR map; + gctBOOL needCopy = gcvFALSE; + gcsSTATE_DELTA_PTR nDelta; + gcsSTATE_DELTA_PTR uDelta = gcvNULL; + gcsSTATE_DELTA_PTR kDelta = gcvNULL; + gcsSTATE_DELTA_RECORD_PTR record; + gcsSTATE_DELTA_RECORD_PTR recordArray = gcvNULL; + gctUINT elementCount; + gctUINT address; + gctUINT32 mask; + gctUINT32 data; + gctUINT index; + gctUINT i, j; + gctUINT32 dirtyRecordArraySize = 0; + + gcmkHEADER_ARG( + "Context=%p ProcessID=%d StateDelta=%p", + Context, ProcessID, StateDelta + ); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Context, gcvOBJ_CONTEXT); + + /* Get a shortcut to the kernel object. */ + kernel = Context->hardware->kernel; + + /* Check wehther we need to copy the structures or not. */ + gcmkONERROR(gckOS_QueryNeedCopy(Context->os, ProcessID, &needCopy)); + + /* Get the current context buffer. */ + buffer = Context->buffer; + + /* Wait until the context buffer becomes available; this will + also reset the signal and mark the buffer as busy. */ + gcmkONERROR(gckOS_WaitSignal( + Context->os, buffer->signal, gcvFALSE, gcvINFINITE + )); + +#if gcmIS_DEBUG(gcdDEBUG_CODE) && 1 && gcdENABLE_3D + /* Update current context token. */ + buffer->logical[Context->map[0x0E14].index] + = (gctUINT32)gcmPTR2INT32(Context); +#endif + + /* Are there any pending deltas? */ + if (buffer->deltaCount != 0) + { + /* Get the state map. */ + map = Context->map; + + /* Get the first delta item. */ + uDelta = buffer->delta; + + /* Reset the vertex stream count. */ + elementCount = 0; + + /* Merge all pending deltas. */ + for (i = 0; i < buffer->deltaCount; i += 1) + { + /* Get access to the state delta. */ + gcmkONERROR(gckKERNEL_OpenUserData( + kernel, needCopy, + &_stateDelta, + uDelta, gcmSIZEOF(gcsSTATE_DELTA), + (gctPOINTER *) &kDelta + )); + + dirtyRecordArraySize + = gcmSIZEOF(gcsSTATE_DELTA_RECORD) * kDelta->recordCount; + + if (dirtyRecordArraySize) + { + /* Get access to the state records. */ + gcmkONERROR(gckOS_MapUserPointer( + kernel->os, + gcmUINT64_TO_PTR(kDelta->recordArray), + dirtyRecordArraySize, + (gctPOINTER *) &recordArray + )); + + if (recordArray == gcvNULL) + { + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + /* Merge all pending states. */ + for (j = 0; j < kDelta->recordCount; j += 1) + { + if (j >= Context->numStates) + { + break; + } + + /* Get the current state record. */ + record = &recordArray[j]; + + /* Get the state address. */ + gcmkONERROR(gckOS_ReadMappedPointer(kernel->os, &record->address, &address)); + + /* Make sure the state is a part of the mapping table. */ + if (address >= Context->maxState) + { + gcmkTRACE( + gcvLEVEL_ERROR, + "%s(%d): State 0x%04X (0x%04X) is not mapped.\n", + __FUNCTION__, __LINE__, + address, address << 2 + ); + + continue; + } + + /* Get the state index. */ + index = map[address].index; + + /* Skip the state if not mapped. */ + if (index == 0) + { + continue; + } + + /* Get the data mask. */ + gcmkONERROR(gckOS_ReadMappedPointer(kernel->os, &record->mask, &mask)); + + /* Get the new data value. */ + gcmkONERROR(gckOS_ReadMappedPointer(kernel->os, &record->data, &data)); + + /* Masked states that are being completly reset or regular states. */ + if ((mask == 0) || (mask == ~0U)) + { + /* Process special states. */ + if (address == 0x0595) + { + /* Force auto-disable to be disabled. */ + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))); + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))); + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 13:13) - (0 ? + 13:13) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 13:13) - (0 ? + 13:13) + 1))))))) << (0 ? + 13:13))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 13:13) - (0 ? + 13:13) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 13:13) - (0 ? 13:13) + 1))))))) << (0 ? 13:13))); + } + + /* Set new data. */ + buffer->logical[index] = data; + } + + /* Masked states that are being set partially. */ + else + { + buffer->logical[index] + = (~mask & buffer->logical[index]) + | (mask & data); + } + } + } + + /* Get the element count. */ + if (kDelta->elementCount != 0) + { + elementCount = kDelta->elementCount; + } + + /* Dereference delta. */ + kDelta->refCount -= 1; + gcmkASSERT(kDelta->refCount >= 0); + + /* Get the next state delta. */ + nDelta = gcmUINT64_TO_PTR(kDelta->next); + + if (dirtyRecordArraySize) + { + /* Get access to the state records. */ + gcmkONERROR(gckOS_UnmapUserPointer( + kernel->os, + gcmUINT64_TO_PTR(kDelta->recordArray), + dirtyRecordArraySize, + recordArray + )); + + recordArray = gcvNULL; + } + + /* Close access to the current state delta. */ + gcmkONERROR(gckKERNEL_CloseUserData( + kernel, needCopy, + gcvTRUE, + uDelta, gcmSIZEOF(gcsSTATE_DELTA), + (gctPOINTER *) &kDelta + )); + + /* Update the user delta pointer. */ + uDelta = nDelta; + } + + /* Hardware disables all input attribute when the attribute 0 is programmed, + it then reenables those attributes that were explicitely programmed by + the software. Because of this we cannot program the entire array of + values, otherwise we'll get all attributes reenabled, but rather program + only those that are actully needed by the software. + elementCount = attribCount + 1 to make sure 0 is a flag to indicate if UMD + touches it. + */ + if (elementCount != 0) + { + gctUINT base; + gctUINT nopCount; + gctUINT32_PTR nop; + gctUINT fe2vsCount; + gctUINT attribCount = elementCount -1; + gctUINT32 feAttributeStatgeAddr = 0x0180; + if (gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_HALTI5)) + { + fe2vsCount = 32; + base = map[0x5E00].index; + feAttributeStatgeAddr = 0x5E00; + } + else if (gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_HALTI0)) + { + fe2vsCount = 16; + base = map[0x0180].index; + } + else + { + fe2vsCount = 12; + base = map[0x0180].index; + } + + /* Set the proper state count. */ + if (attribCount == 0) + { + gcmkASSERT(gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_ZERO_ATTRIB_SUPPORT)); + + buffer->logical[base - 1] + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 26:26) - (0 ? + 26:26) + 1))))))) << (0 ? + 26:26))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 26:26) - (0 ? 26:26) + 1))))))) << (0 ? 26:26))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (feAttributeStatgeAddr) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + /* Set the proper state count. */ + buffer->logical[base + 1] = + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 26:26) - (0 ? + 26:26) + 1))))))) << (0 ? + 26:26))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 26:26) - (0 ? 26:26) + 1))))))) << (0 ? 26:26))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x01F2) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + buffer->logical[base + 2] = 0x1; + attribCount = 3; + } + else + { + buffer->logical[base - 1] + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 26:26) - (0 ? + 26:26) + 1))))))) << (0 ? + 26:26))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 26:26) - (0 ? + 26:26) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 26:26) - (0 ? 26:26) + 1))))))) << (0 ? 26:26))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (attribCount) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (feAttributeStatgeAddr) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + } + + /* Determine the number of NOP commands. */ + nopCount = (fe2vsCount / 2) - (attribCount / 2); + /* Determine the location of the first NOP. */ + nop = &buffer->logical[base + (attribCount | 1)]; + + /* Fill the unused space with NOPs. */ + for (i = 0; i < nopCount; i += 1) + { + if (nop >= buffer->logical + Context->totalSize) + { + break; + } + + /* Generate a NOP command. */ + *nop = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + /* Advance. */ + nop += 2; + } + } + + if (gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_SMALL_BATCH) && + Context->hardware->options.smallBatch) + { + gctUINT numConstant = (gctUINT)Context->hardware->identity.numConstants; + gctUINT32 constCount = 0; + + /* Get the const number after merge. */ + index = map[0x042B].index; + data = buffer->logical[index]; + constCount = (((((gctUINT32) (data)) >> (0 ? 8:0)) & ((gctUINT32) ((((1 ? 8:0) - (0 ? 8:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 8:0) - (0 ? 8:0) + 1)))))) ); + + _UpdateUnifiedReg(Context, 0xD000, numConstant << 2, constCount << 2); + } + + if (gckHARDWARE_IsFeatureAvailable(Context->hardware, gcvFEATURE_SMALL_BATCH) && + Context->hardware->options.smallBatch) + { + gctUINT numSamplers = 80; + gctUINT32 samplerCount = 0; + + /* Get the sampler number after merge. */ + index = map[0x042C].index; + data = buffer->logical[index]; + samplerCount = (((((gctUINT32) (data)) >> (0 ? 6:0)) & ((gctUINT32) ((((1 ? 6:0) - (0 ? 6:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 6:0) - (0 ? 6:0) + 1)))))) ); + + _UpdateUnifiedReg(Context, 0x5800, numSamplers, samplerCount); + _UpdateUnifiedReg(Context, 0x5880, numSamplers, samplerCount); + _UpdateUnifiedReg(Context, 0x5900, numSamplers, samplerCount); + _UpdateUnifiedReg(Context, 0x5980, numSamplers, samplerCount); + _UpdateUnifiedReg(Context, 0x5A00, numSamplers, samplerCount); + _UpdateUnifiedReg(Context, 0x5600, numSamplers, samplerCount); + _UpdateUnifiedReg(Context, 0x5680, numSamplers, samplerCount); + } + /* Reset pending deltas. */ + buffer->deltaCount = 0; + buffer->delta = gcvNULL; + } + + if (StateDelta) + { + /* Set state delta user pointer. */ + uDelta = StateDelta; + + /* Get access to the state delta. */ + gcmkONERROR(gckKERNEL_OpenUserData( + kernel, needCopy, + &_stateDelta, + uDelta, gcmSIZEOF(gcsSTATE_DELTA), + (gctPOINTER *) &kDelta + )); + + /* State delta cannot be attached to anything yet. */ + if (kDelta->refCount != 0) + { + gcmkTRACE( + gcvLEVEL_ERROR, + "%s(%d): kDelta->refCount = %d (has to be 0).\n", + __FUNCTION__, __LINE__, + kDelta->refCount + ); + } + + /* Attach to all contexts. */ + buffer = Context->buffer; + + do + { + /* Attach to the context if nothing is attached yet. If a delta + is allready attached, all we need to do is to increment + the number of deltas in the context. */ + if (buffer->delta == gcvNULL) + { + buffer->delta = uDelta; + } + + /* Update reference count. */ + kDelta->refCount += 1; + + /* Update counters. */ + buffer->deltaCount += 1; + + /* Get the next context buffer. */ + buffer = buffer->next; + + if (buffer == gcvNULL) + { + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + } + while (Context->buffer != buffer); + + /* Close access to the current state delta. */ + gcmkONERROR(gckKERNEL_CloseUserData( + kernel, needCopy, + gcvTRUE, + uDelta, gcmSIZEOF(gcsSTATE_DELTA), + (gctPOINTER *) &kDelta + )); + + } + /* Schedule an event to mark the context buffer as available. */ + gcmkONERROR(gckEVENT_Signal( + buffer->eventObj, buffer->signal, gcvKERNEL_PIXEL + )); + + /* Advance to the next context buffer. */ + Context->buffer = buffer->next; + + /* Return the status. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + /* Get access to the state records. */ + if (kDelta != gcvNULL && recordArray != gcvNULL) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + kernel->os, + gcmUINT64_TO_PTR(kDelta->recordArray), + dirtyRecordArraySize, + (gctPOINTER *) &recordArray + )); + } + + /* Close access to the current state delta. */ + gcmkVERIFY_OK(gckKERNEL_CloseUserData( + kernel, needCopy, + gcvTRUE, + uDelta, gcmSIZEOF(gcsSTATE_DELTA), + (gctPOINTER *) &kDelta + )); + + /* Return the status. */ + gcmkFOOTER(); + return status; +#else + return gcvSTATUS_OK; +#endif +} + +gceSTATUS +gckCONTEXT_MapBuffer( + IN gckCONTEXT Context, + OUT gctUINT64 *Logicals, + OUT gctUINT32 *Bytes + ) +{ + gceSTATUS status; + int i = 0; + gckKERNEL kernel = Context->hardware->kernel; + gctPOINTER logical; + gcsCONTEXT_PTR buffer; + + gcmkHEADER_ARG("Context=%p", Context); + + buffer = Context->buffer; + + for (i = 0; i < gcdCONTEXT_BUFFER_COUNT; i++) + { + /* Lock for userspace CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + kernel, + buffer->videoMem, + gcvFALSE, + gcvTRUE, + &logical + )); + + Logicals[i] = gcmPTR_TO_UINT64(logical); + buffer = buffer->next; + } + + *Bytes = (gctUINT)Context->totalSize; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} +
diff --git a/hal/kernel/arch/gc_hal_kernel_context.h b/hal/kernel/arch/gc_hal_kernel_context.h new file mode 100644 index 0000000..5f99ddf --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_context.h
@@ -0,0 +1,193 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_kernel_context_h_ +#define __gc_hal_kernel_context_h_ + +#include "gc_hal_kernel_buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Maps state locations within the context buffer. */ +typedef struct _gcsSTATE_MAP * gcsSTATE_MAP_PTR; +typedef struct _gcsSTATE_MAP +{ + /* Index of the state in the context buffer. */ + gctUINT index; + + /* State mask. */ + gctUINT32 mask; +} +gcsSTATE_MAP; + +/* Context buffer. */ +typedef struct _gcsCONTEXT * gcsCONTEXT_PTR; +typedef struct _gcsCONTEXT +{ + /* For debugging: the number of context buffer in the order of creation. */ + gctUINT num; + + /* Pointer to gckEVENT object. */ + gckEVENT eventObj; + + /* Context busy signal. */ + gctSIGNAL signal; + + /* Video memory of the context buffer. */ + gckVIDMEM_NODE videoMem; + +#if gcdCAPTURE_ONLY_MODE + gctUINT32 handle; +#endif + + /* Logical address of the context buffer. */ + gctUINT32_PTR logical; + + /* Hardware address of the context buffer. */ + gctUINT32 address; + + /* Pointer to the LINK commands. */ + gctPOINTER link2D; + gctPOINTER link3D; + + /* The number of pending state deltas. */ + gctUINT deltaCount; + + /* Pointer to the first delta to be applied. */ + gcsSTATE_DELTA_PTR delta; + + /* Next context buffer. */ + gcsCONTEXT_PTR next; +} +gcsCONTEXT; + +typedef struct _gcsRECORD_ARRAY_MAP * gcsRECORD_ARRAY_MAP_PTR; +struct _gcsRECORD_ARRAY_MAP +{ + /* User pointer key. */ + gctUINT64 key; + + /* Kernel memory buffer. */ + gcsSTATE_DELTA_RECORD_PTR kData; + + /* Next map. */ + gcsRECORD_ARRAY_MAP_PTR next; + +}; + +#define USE_SW_RESET 1 + +/* gckCONTEXT structure that hold the current context. */ +struct _gckCONTEXT +{ + /* Object. */ + gcsOBJECT object; + + /* Pointer to gckOS object. */ + gckOS os; + + /* Pointer to gckHARDWARE object. */ + gckHARDWARE hardware; + + /* Command buffer alignment. */ + gctUINT32 alignment; + gctUINT32 reservedHead; + + /* Context buffer metrics. */ + gctSIZE_T maxState; + gctUINT32 numStates; + gctUINT32 totalSize; + gctUINT32 bufferSize; + gctUINT32 linkIndex2D; + gctUINT32 linkIndex3D; + gctUINT32 linkIndexXD; + gctUINT32 entryOffset3D; + gctUINT32 entryOffsetXDFrom2D; + gctUINT32 entryOffsetXDFrom3D; + + /* State mapping. */ + gcsSTATE_MAP_PTR map; + + /* List of context buffers. */ + gcsCONTEXT_PTR buffer; + + /* Requested pipe select for context. */ + gcePIPE_SELECT entryPipe; + gcePIPE_SELECT exitPipe; + + /* Variables used for building state buffer. */ + gctUINT32 lastAddress; + gctSIZE_T lastSize; + gctUINT32 lastIndex; + gctBOOL lastFixed; + + gctUINT32 pipeSelectBytes; + + gcsPROFILER_COUNTERS_PART1 latestProfiler_part1; + gcsPROFILER_COUNTERS_PART1 histroyProfiler_part1; + gcsPROFILER_COUNTERS_PART1 preProfiler_part1; + gcsPROFILER_COUNTERS_PART2 latestProfiler_part2; + gcsPROFILER_COUNTERS_PART2 histroyProfiler_part2; + gcsPROFILER_COUNTERS_PART2 preProfiler_part2; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* __gc_hal_kernel_context_h_ */ +
diff --git a/hal/kernel/arch/gc_hal_kernel_hardware.c b/hal/kernel/arch/gc_hal_kernel_hardware.c new file mode 100644 index 0000000..b1bf686 --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_hardware.c
@@ -0,0 +1,15954 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal.h" +#include "gc_hal_kernel.h" +#include "gc_hal_kernel_context.h" + +#include "gc_feature_database.h" +#include <gc_hal_kernel_debug.h> + +#define _GC_OBJ_ZONE gcvZONE_HARDWARE + +typedef struct _gcsiDEBUG_REGISTERS * gcsiDEBUG_REGISTERS_PTR; +typedef struct _gcsiDEBUG_REGISTERS +{ + gctSTRING module; + gctUINT index; + gctUINT shift; + gctUINT data; + gctUINT count; + gctUINT32 pipeMask; + gctUINT32 selectStart; + gctBOOL avail; + gctBOOL inCluster; +} +gcsiDEBUG_REGISTERS; + +typedef struct _gcsFE_STACK +{ + gctSTRING name; + gctINT count; + gctUINT32 highSelect; + gctUINT32 lowSelect; + gctUINT32 linkSelect; + gctUINT32 clear; + gctUINT32 next; +} +gcsFE_STACK; + +/******************************************************************************\ +********************************* Support Code ********************************* +\******************************************************************************/ +static gctBOOL +_IsHardwareMatch( + IN gckHARDWARE Hardware, + IN gctINT32 ChipModel, + IN gctUINT32 ChipRevision + ) +{ + return ((Hardware->identity.chipModel == ChipModel) && + (Hardware->identity.chipRevision == ChipRevision)); +} + +static gceSTATUS +_ResetGPU( + IN gckHARDWARE Hardware, + IN gckOS Os, + IN gceCORE Core + ); + +static void +_GetEcoID( + IN gckHARDWARE Hardware, + IN OUT gcsHAL_QUERY_CHIP_IDENTITY_PTR Identity + ) +{ + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x000E8, + &Identity->ecoID + )); + + if (_IsHardwareMatch(Hardware, 0x1000, 0x5037) && (Identity->chipDate == 0x20120617)) + { + Identity->ecoID = 1; + } + + if (_IsHardwareMatch(Hardware, 0x320, 0x5303) && (Identity->chipDate == 0x20140511)) + { + Identity->ecoID = 1; + } + +} + +static gceSTATUS +_IdentifyHardwareByDatabase( + IN gckHARDWARE Hardware, + IN gckOS Os, + IN gckDEVICE Device, + IN gceCORE Core, + OUT gcsHAL_QUERY_CHIP_IDENTITY_PTR Identity + ) +{ + gceSTATUS status; + gctUINT32 chipIdentity; + gctUINT32 debugControl0; + gctUINT32 chipInfo; + gcsFEATURE_DATABASE *database; + gctUINT i = 0; + + gcmkHEADER_ARG("Os=0x%x", Os); + + /* Get chip date. */ + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00028, &Identity->chipDate)); + + /*************************************************************************** + ** Get chip ID and revision. + */ + + /* Read chip identity register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00018, + &chipIdentity)); + + /* Special case for older graphic cores. */ + if (((((gctUINT32) (chipIdentity)) >> (0 ? + 31:24) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1)))))) == (0x01 & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1)))))))) + { + Identity->chipModel = gcv500; + Identity->chipRevision = (((((gctUINT32) (chipIdentity)) >> (0 ? 15:12)) & ((gctUINT32) ((((1 ? 15:12) - (0 ? 15:12) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 15:12) - (0 ? 15:12) + 1)))))) ); + } + + else + { + /* Read chip identity register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00020, + (gctUINT32_PTR) &Identity->chipModel)); + + if (((Identity->chipModel & 0xFF00) == 0x0400) + && (Identity->chipModel != 0x0420) + && (Identity->chipModel != 0x0428)) + { + Identity->chipModel = (gceCHIPMODEL) (Identity->chipModel & 0x0400); + } + + /* Read CHIP_REV register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00024, + &Identity->chipRevision)); + + if ((Identity->chipModel == gcv300) + && (Identity->chipRevision == 0x2201) + ) + { + gctUINT32 chipDate; + gctUINT32 chipTime; + + /* Read date and time registers. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00028, + &chipDate)); + + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x0002C, + &chipTime)); + + if ((chipDate == 0x20080814) && (chipTime == 0x12051100)) + { + /* This IP has an ECO; put the correct revision in it. */ + Identity->chipRevision = 0x1051; + } + } + + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x000A8, + &Identity->productID)); + } + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "Identity: chipModel=%X", + Identity->chipModel); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "Identity: chipRevision=%X", + Identity->chipRevision); + + _GetEcoID(Hardware, Identity); + + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00030, + &Identity->customerID)); + + /*get hw minor features*/ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x0001C, + &Identity->chipFeatures)); + + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00034, + &Identity->chipMinorFeatures)); + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00074, + &Identity->chipMinorFeatures1)); + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00084, + &Identity->chipMinorFeatures2)); + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00088, + &Identity->chipMinorFeatures3)); + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00094, + &Identity->chipMinorFeatures4)); + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x000A0, + &Identity->chipMinorFeatures5)); + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x000DC, + &Identity->chipMinorFeatures6)); + + /*************************************************************************** + ** Get chip features. + */ + + database = + Hardware->featureDatabase = + gcQueryFeatureDB( + Hardware->identity.chipModel, + Hardware->identity.chipRevision, + Hardware->identity.productID, + Hardware->identity.ecoID, + Hardware->identity.customerID); + + if (database == gcvNULL) + { + gcmkPRINT("[galcore]: Feature database is not found," + "chipModel=0x%0x, chipRevision=0x%x, productID=0x%x, ecoID=0x%x, customerID=0x%x", + Hardware->identity.chipModel, + Hardware->identity.chipRevision, + Hardware->identity.productID, + Hardware->identity.ecoID, + Hardware->identity.customerID); + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + else if (database->chipVersion != Hardware->identity.chipRevision) + { + gcmkPRINT("[galcore]: Warning: chipRevision mismatch, database chipRevision=0x%x register read chipRevision=0x%x\n", + database->chipVersion, Hardware->identity.chipRevision); + } + + + Identity->pixelPipes = database->NumPixelPipes; + Identity->resolvePipes = database->NumResolvePipes; + Identity->instructionCount = database->InstructionCount; + Identity->numConstants = database->NumberOfConstants; + Identity->varyingsCount = database->VaryingCount; + Identity->gpuCoreCount = database->CoreCount; + Identity->streamCount = database->Streams; + Identity->clusterAvailMask = database->ClusterAliveMask; + + if (gcmIS_SUCCESS(gckOS_QueryOption(Hardware->os, "sRAMBases", Device->sRAMBases[0]))) + { + gckOS_MemCopy( + Identity->sRAMBases, + Device->sRAMBases[Core], + sizeof(gctUINT64) * gcvSRAM_INTER_COUNT + ); + } + else + { + for (i = 0; i < gcvSRAM_INTER_COUNT; i++) + { + Identity->sRAMBases[i] = gcvINVALID_PHYSICAL_ADDRESS; + } + } + + if (gcmIS_SUCCESS(gckOS_QueryOption(Hardware->os, "sRAMSizes", (gctUINT64 *)Device->sRAMSizes[0]))) + { + gckOS_MemCopy( + Identity->sRAMSizes, + Device->sRAMSizes[Core], + sizeof(gctUINT32) * gcvSRAM_INTER_COUNT + ); + } + + for (i = gcvSRAM_INTERNAL0; i < gcvSRAM_INTER_COUNT; i++) + { + if (Identity->sRAMSizes[i]) + { + break; + } + } + + /* If module parameter doesn't set per-core SRAM sizes. */ + if (i == gcvSRAM_INTER_COUNT) + { + gctUINT j = 0; + for (i = Core; i < gcvCORE_COUNT; i++) + { + for (j = gcvSRAM_INTERNAL0; j < gcvSRAM_INTER_COUNT; j++) + { + /* Try to get SRAM sizes from database. */ + Device->sRAMSizes[i][j] = Identity->sRAMSizes[j] = database->VIP_SRAM_SIZE; + } + } + } + + gckOS_QueryOption(Hardware->os, "extSRAMBases", Device->extSRAMBases); + + gckOS_QueryOption(Hardware->os, "extSRAMSizes", (gctUINT64 *)Device->extSRAMSizes); + + for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++) + { + if (Device->extSRAMSizes[i]) + { + break; + } + } + + /* If module parameter doesn't set external SRAM sizes. */ + if (i == gcvSRAM_EXT_COUNT) + { + for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++) + { + /* Try to get SRAM sizes from database. */ + Device->extSRAMSizes[i] = database->AXI_SRAM_SIZE; + } + } + + if (Identity->chipModel == gcv320) + { + gctUINT32 data; + + gcmkONERROR( + gckOS_ReadRegisterEx(Os, + Core, + 0x0002C, + &data)); + + if ((data != 33956864) && + ((Identity->chipRevision == 0x5007) || + (Identity->chipRevision == 0x5220))) + { + Hardware->maxOutstandingReads = 0xFF & + (Identity->chipRevision == 0x5220 ? 8 : + (Identity->chipRevision == 0x5007 ? 12 : 0)); + } + } + + if (_IsHardwareMatch(Hardware, gcv880, 0x5107)) + { + Hardware->maxOutstandingReads = 0x00010; + } + + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00470, &debugControl0)); + + if (debugControl0 & (1 << 16)) + { + Identity->chipFlags |= gcvCHIP_FLAG_MSAA_COHERENCEY_ECO_FIX; + } + + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x000A4, &chipInfo)); + + if (((((gctUINT32) (chipInfo)) >> (0 ? + 21:21) & ((gctUINT32) ((((1 ? + 21:21) - (0 ? + 21:21) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 21:21) - (0 ? + 21:21) + 1)))))) == (0x1 & ((gctUINT32) ((((1 ? + 21:21) - (0 ? + 21:21) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 21:21) - (0 ? 21:21) + 1)))))))) + { + Identity->chipFlags |= gcvCHIP_AXI_BUS128_BITS; + } + + gckOS_QueryOption(Os, "platformFlagBits", &Identity->platformFlagBits); + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_GetHardwareSignature( + IN gckHARDWARE Hardware, + IN gckOS Os, + IN gceCORE Core, + OUT gcsHARDWARE_SIGNATURE * Signature + ) +{ + gceSTATUS status; + + gctUINT32 chipIdentity; + + gcmkHEADER_ARG("Os=0x%x", Os); + + /*************************************************************************** + ** Get chip ID and revision. + */ + + /* Read chip identity register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00018, + &chipIdentity)); + + /* Special case for older graphic cores. */ + if (((((gctUINT32) (chipIdentity)) >> (0 ? + 31:24) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1)))))) == (0x01 & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1)))))))) + { + Signature->chipModel = gcv500; + Signature->chipRevision = (((((gctUINT32) (chipIdentity)) >> (0 ? 15:12)) & ((gctUINT32) ((((1 ? 15:12) - (0 ? 15:12) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 15:12) - (0 ? 15:12) + 1)))))) ); + } + + else + { + /* Read chip identity register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00020, + (gctUINT32_PTR) &Signature->chipModel)); + + /* Read CHIP_REV register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00024, + &Signature->chipRevision)); + } + + /*************************************************************************** + ** Get chip features. + */ + + /* Read chip feature register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x0001C, + &Signature->chipFeatures)); + + if (((Signature->chipModel == gcv500) && (Signature->chipRevision < 2)) + || ((Signature->chipModel == gcv300) && (Signature->chipRevision < 0x2000)) + ) + { + /* GC500 rev 1.x and GC300 rev < 2.0 doesn't have these registers. */ + Signature->chipMinorFeatures = 0; + Signature->chipMinorFeatures1 = 0; + Signature->chipMinorFeatures2 = 0; + } + else + { + /* Read chip minor feature register #0. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00034, + &Signature->chipMinorFeatures)); + + if (((((gctUINT32) (Signature->chipMinorFeatures)) >> (0 ? + 21:21) & ((gctUINT32) ((((1 ? + 21:21) - (0 ? + 21:21) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 21:21) - (0 ? + 21:21) + 1)))))) == (0x1 & ((gctUINT32) ((((1 ? + 21:21) - (0 ? + 21:21) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 21:21) - (0 ? 21:21) + 1))))))) + ) + { + /* Read chip minor features register #1. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00074, + &Signature->chipMinorFeatures1)); + + /* Read chip minor features register #2. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Os, Core, + 0x00084, + &Signature->chipMinorFeatures2)); + } + else + { + /* Chip doesn't has minor features register #1 or 2 or 3 or 4 or 5. */ + Signature->chipMinorFeatures1 = 0; + Signature->chipMinorFeatures2 = 0; + } + } + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/* Set to 1 to enable module clock gating debug function. +* Following options take effect when it is set to 1. +*/ +#define gcdDEBUG_MODULE_CLOCK_GATING 0 +/* Set to 1 to disable module clock gating of all modules. */ +#define gcdDISABLE_MODULE_CLOCK_GATING 0 +/* Set to 1 to disable module clock gating of each module. */ +#define gcdDISABLE_STARVE_MODULE_CLOCK_GATING 0 +#define gcdDISABLE_FE_CLOCK_GATING 0 +#define gcdDISABLE_PE_CLOCK_GATING 0 +#define gcdDISABLE_SH_CLOCK_GATING 0 +#define gcdDISABLE_PA_CLOCK_GATING 0 +#define gcdDISABLE_SE_CLOCK_GATING 0 +#define gcdDISABLE_RA_CLOCK_GATING 0 +#define gcdDISABLE_RA_EZ_CLOCK_GATING 0 +#define gcdDISABLE_RA_HZ_CLOCK_GATING 0 +#define gcdDISABLE_TX_CLOCK_GATING 0 +#define gcdDISABLE_TFB_CLOCK_GATING 0 +#define gcdDISABLE_GPIPE_CLOCK_GATING 0 +#define gcdDISABLE_BLT_CLOCK_GATING 0 +#define gcdDISABLE_TPG_CLOCK_GATING 0 +#define gcdDISABLE_VX_CLOCK_GATING 0 + +#if gcdDEBUG_MODULE_CLOCK_GATING +gceSTATUS +_ConfigureModuleLevelClockGating( + gckHARDWARE Hardware + ) +{ + gctUINT32 data; + + gcmkVERIFY_OK( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + &data)); + +#if gcdDISABLE_FE_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); +#endif + +#if gcdDISABLE_PE_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2))); +#endif + +#if gcdDISABLE_SH_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))); +#endif + +#if gcdDISABLE_PA_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))); +#endif + +#if gcdDISABLE_SE_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))); +#endif + +#if gcdDISABLE_RA_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:6) - (0 ? + 6:6) + 1))))))) << (0 ? + 6:6))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:6) - (0 ? 6:6) + 1))))))) << (0 ? 6:6))); +#endif + +#if gcdDISABLE_TX_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7))); +#endif + +#if gcdDISABLE_RA_EZ_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))); +#endif + +#if gcdDISABLE_RA_HZ_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:17) - (0 ? + 17:17) + 1))))))) << (0 ? + 17:17))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:17) - (0 ? 17:17) + 1))))))) << (0 ? 17:17))); +#endif + +#if gcdDISABLE_TFB_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 19:19) - (0 ? + 19:19) + 1))))))) << (0 ? + 19:19))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 19:19) - (0 ? 19:19) + 1))))))) << (0 ? 19:19))); +#endif + +#if gcdDISABLE_GPIPE_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 22:22) - (0 ? + 22:22) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 22:22) - (0 ? + 22:22) + 1))))))) << (0 ? + 22:22))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 22:22) - (0 ? + 22:22) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 22:22) - (0 ? 22:22) + 1))))))) << (0 ? 22:22))); +#endif + +#if gcdDISABLE_BLT_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 20:20) - (0 ? + 20:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 20:20) - (0 ? + 20:20) + 1))))))) << (0 ? + 20:20))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 20:20) - (0 ? + 20:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 20:20) - (0 ? 20:20) + 1))))))) << (0 ? 20:20))); +#endif + +#if gcdDISABLE_TPG_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 18:18) - (0 ? + 18:18) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 18:18) - (0 ? + 18:18) + 1))))))) << (0 ? + 18:18))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 18:18) - (0 ? + 18:18) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 18:18) - (0 ? 18:18) + 1))))))) << (0 ? 18:18))); +#endif + +#if gcdDISABLE_VX_CLOCK_GATING + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 21:21) - (0 ? + 21:21) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 21:21) - (0 ? + 21:21) + 1))))))) << (0 ? + 21:21))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 21:21) - (0 ? + 21:21) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 21:21) - (0 ? 21:21) + 1))))))) << (0 ? 21:21))); +#endif + + gcmkVERIFY_OK( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + data)); + +#if gcdDISABLE_STARVE_MODULE_CLOCK_GATING + gcmkVERIFY_OK( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00100, + &data)); + + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2))); + + gcmkVERIFY_OK( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00100, + data)); + +#endif + +#if gcdDISABLE_MODULE_CLOCK_GATING + gcmkVERIFY_OK( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00100, + &data)); + + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + + gcmkVERIFY_OK( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00100, + data)); +#endif + + return gcvSTATUS_OK; +} +#endif + +static void +_PowerStateTimerFunc( + gctPOINTER Data + ) +{ + gckHARDWARE hardware = (gckHARDWARE)Data; + + gcmkVERIFY_OK(gckHARDWARE_SetPowerState(hardware, hardware->nextPowerState)); +} + +static gceSTATUS +_VerifyDMA( + IN gckOS Os, + IN gceCORE Core, + gctUINT32_PTR Address1, + gctUINT32_PTR Address2, + gctUINT32_PTR State1, + gctUINT32_PTR State2 + ) +{ + gceSTATUS status; + gctUINT32 i; + + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00660, State1)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00660, State1)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00664, Address1)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00664, Address1)); + + for (i = 0; i < 500; i += 1) + { + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00660, State2)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00660, State2)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00664, Address2)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00664, Address2)); + + if (*Address1 != *Address2) + { + break; + } + + if (*State1 != *State2) + { + break; + } + } + +OnError: + return status; +} + +static gceSTATUS +_DumpDebugRegisters( + IN gckOS Os, + IN gceCORE Core, + IN gcsiDEBUG_REGISTERS_PTR Descriptor + ) +{ +/* If this value is changed, print formats need to be changed too. */ +#define REG_PER_LINE 8 + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 select; + gctUINT i, j, pipe; + gctUINT32 datas[REG_PER_LINE]; + gctUINT32 oldControl, control; + + gcmkHEADER_ARG("Os=0x%X Descriptor=0x%X", Os, Descriptor); + + /* Record control. */ + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x0, &oldControl)); + + for (pipe = 0; pipe < 4; pipe++) + { + if (!Descriptor->avail) + { + continue; + } + if (!(Descriptor->pipeMask & (1 << pipe))) + { + continue; + } + + gcmkPRINT_N(8, " %s[%d] debug registers:\n", Descriptor->module, pipe); + + /* Switch pipe. */ + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x0, &control)); + control &= ~(0xF << 20); + control |= (pipe << 20); + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x0, control)); + + gcmkASSERT(!(Descriptor->count % REG_PER_LINE)); + + for (i = 0; i < Descriptor->count; i += REG_PER_LINE) + { + /* Select of first one in the group. */ + select = i + Descriptor->selectStart; + + /* Read a group of registers. */ + for (j = 0; j < REG_PER_LINE; j++) + { + /* Shift select to right position. */ + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, (select + j) << Descriptor->shift)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, Descriptor->data, &datas[j])); + } + + gcmkPRINT_N(32, " [%02X] %08X %08X %08X %08X %08X %08X %08X %08X\n", + select, datas[0], datas[1], datas[2], datas[3], datas[4], datas[5], datas[6], datas[7]); + } + } + + /* Restore control. */ + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x0, oldControl)); + +OnError: + /* Return the error. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_DumpLinkStack( + IN gckOS Os, + IN gceCORE Core, + IN gcsiDEBUG_REGISTERS_PTR Descriptor + ) +{ + /* Get wrptr */ + gctUINT32 shift = Descriptor->shift; + gctUINT32 pointerSelect = 0xE << shift; + gctUINT32 pointer, wrPtr, rdPtr, links[16]; + gctUINT32 stackSize = 16; + gctUINT32 oldestPtr = 0; + gctUINT32 i; + + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, pointerSelect)); + gcmkVERIFY_OK(gckOS_ReadRegisterEx(Os, Core, Descriptor->data, &pointer)); + + wrPtr = (pointer & 0xF0) >> 4; + rdPtr = pointer & 0xF; + + /* Move rdptr to the oldest one (next one to the latest one. ) */ + oldestPtr = (wrPtr + 1) % stackSize; + + while (rdPtr != oldestPtr) + { + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, 0x0)); + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, 0xF << shift)); + + + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, pointerSelect)); + gcmkVERIFY_OK(gckOS_ReadRegisterEx(Os, Core, Descriptor->data, &pointer)); + + rdPtr = pointer & 0xF; + } + + gcmkPRINT(" Link stack:"); + + /* Read from stack bottom*/ + for (i = 0; i < stackSize; i++) + { + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, 0xD << shift)); + gcmkVERIFY_OK(gckOS_ReadRegisterEx(Os, Core, Descriptor->data, &links[i])); + + /* Advance rdPtr. */ + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, 0x0)); + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, 0xF << shift)); + } + + /* Print. */ + for (i = 0; i < stackSize; i += 4) + { + gcmkPRINT_N(32, " [0x%02X] 0x%08X [0x%02X] 0x%08X [0x%02X] 0x%08X [0x%02X] 0x%08X\n", + i, links[i], i + 1, links[i + 1], i + 2, links[i + 2], i + 3, links[i + 3]); + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_DumpFEStack( + IN gckOS Os, + IN gceCORE Core, + IN gcsiDEBUG_REGISTERS_PTR Descriptor + ) +{ + gctUINT i; + gctINT j; + gctUINT32 stack[32][2]; + gctUINT32 link[32]; + + static gcsFE_STACK _feStacks[] = + { + { "PRE_STACK", 32, 0x1A, 0x9A, 0x00, 0x1B, 0x1E }, + { "CMD_STACK", 32, 0x1C, 0x9C, 0x1E, 0x1D, 0x1E }, + }; + + for (i = 0; i < gcmCOUNTOF(_feStacks); i++) + { + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, _feStacks[i].clear)); + + for (j = 0; j < _feStacks[i].count; j++) + { + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, _feStacks[i].highSelect)); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx(Os, Core, Descriptor->data, &stack[j][0])); + + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, _feStacks[i].lowSelect)); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx(Os, Core, Descriptor->data, &stack[j][1])); + + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, _feStacks[i].next)); + + if (_feStacks[i].linkSelect) + { + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Os, Core, Descriptor->index, _feStacks[i].linkSelect)); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx(Os, Core, Descriptor->data, &link[j])); + } + } + + gcmkPRINT(" %s:", _feStacks[i].name); + + for (j = 31; j >= 3; j -= 4) + { + gcmkPRINT(" %08X %08X %08X %08X %08X %08X %08X %08X", + stack[j][0], stack[j][1], stack[j - 1][0], stack[j - 1][1], + stack[j - 2][0], stack[j - 2][1], stack[j - 3][0], stack[j - 3][1]); + } + + if (_feStacks[i].linkSelect) + { + gcmkPRINT(" LINK_STACK:"); + + for (j = 31; j >= 3; j -= 4) + { + gcmkPRINT(" %08X %08X %08X %08X %08X %08X %08X %08X", + link[j], link[j], link[j - 1], link[j - 1], + link[j - 2], link[j - 2], link[j - 3], link[j - 3]); + } + } + + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_IsGPUPresent( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gcsHARDWARE_SIGNATURE signature; + gctUINT32 control; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + &control)); + + control = ((((gctUINT32) (control)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))); + control = ((((gctUINT32) (control)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + control)); + + gckOS_ZeroMemory((gctPOINTER)&signature, gcmSIZEOF(gcsHARDWARE_SIGNATURE)); + + /* Identify the hardware. */ + gcmkONERROR(_GetHardwareSignature(Hardware, + Hardware->os, + Hardware->core, + &signature)); + + /* Check if these are the same values as saved before. */ + if ((Hardware->signature.chipModel != signature.chipModel) + || (Hardware->signature.chipRevision != signature.chipRevision) + || (Hardware->signature.chipFeatures != signature.chipFeatures) + || (Hardware->signature.chipMinorFeatures != signature.chipMinorFeatures) + || (Hardware->signature.chipMinorFeatures1 != signature.chipMinorFeatures1) + || (Hardware->signature.chipMinorFeatures2 != signature.chipMinorFeatures2) + ) + { + gcmkPRINT("[galcore]: GPU is not present."); + gcmkONERROR(gcvSTATUS_GPU_NOT_RESPONDING); + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the error. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_FlushCache( + gckHARDWARE Hardware, + gckCOMMAND Command + ) +{ + gceSTATUS status; + gctUINT32 bytes, requested; + gctPOINTER buffer; + + /* Get the size of the flush command. */ + gcmkONERROR(gckHARDWARE_Flush(Hardware, + gcvFLUSH_ALL, + gcvNULL, + &requested)); + + /* Reserve space in the command queue. */ + gcmkONERROR(gckCOMMAND_Reserve(Command, + requested, + &buffer, + &bytes)); + + /* Append a flush. */ + gcmkONERROR(gckHARDWARE_Flush( + Hardware, gcvFLUSH_ALL, buffer, &bytes + )); + + /* Execute the command queue. */ + gcmkONERROR(gckCOMMAND_Execute(Command, requested)); + + return gcvSTATUS_OK; + +OnError: + return status; +} + +static gctBOOL +_IsGPUIdle( + IN gctUINT32 Idle + ) +{ + return Idle == 0x7FFFFFFF; +} + +static gctBOOL +_QueryFeatureDatabase( + IN gckHARDWARE Hardware, + IN gceFEATURE Feature + ) +{ + gctBOOL available; + + gcsFEATURE_DATABASE *database = Hardware->featureDatabase; + + gcmkHEADER_ARG("Hardware=0x%x Feature=%d", Hardware, Feature); + + /* Only features needed by common kernel logic added here. */ + switch (Feature) + { + case gcvFEATURE_END_EVENT: + available = gcvFALSE; + break; + + case gcvFEATURE_MC20: + available = database->REG_MC20; + break; + + case gcvFEATURE_EARLY_Z: + available = database->REG_NoEZ == 0; + break; + + case gcvFEATURE_HZ: + available = database->REG_HierarchicalZ; + break; + + case gcvFEATURE_NEW_HZ: + available = database->REG_NewHZ; + break; + + case gcvFEATURE_FAST_MSAA: + available = database->REG_FastMSAA; + break; + + case gcvFEATURE_SMALL_MSAA: + available = database->REG_SmallMSAA; + break; + + case gcvFEATURE_DYNAMIC_FREQUENCY_SCALING: + /* This feature doesn't apply for 2D cores. */ + available = database->REG_DynamicFrequencyScaling && database->REG_Pipe3D; + + if (Hardware->identity.chipModel == gcv1000 && + (Hardware->identity.chipRevision == 0x5039 || + Hardware->identity.chipRevision == 0x5040)) + { + available = gcvFALSE; + } + break; + + case gcvFEATURE_ACE: + available = database->REG_ACE; + break; + + case gcvFEATURE_HALTI2: + available = database->REG_Halti2; + break; + + case gcvFEATURE_PIPE_2D: + available = database->REG_Pipe2D; + break; + + case gcvFEATURE_PIPE_3D: +#if gcdENABLE_3D + available = database->REG_Pipe3D; +#else + available = gcvFALSE; +#endif + break; + + case gcvFEATURE_FC_FLUSH_STALL: + available = database->REG_FcFlushStall; + break; + + case gcvFEATURE_BLT_ENGINE: + available = database->REG_BltEngine; + break; + + case gcvFEATURE_HALTI0: + available = database->REG_Halti0; + break; + + case gcvFEATURE_FE_ALLOW_STALL_PREFETCH_ENG: + available = database->REG_FEAllowStallPrefetchEng; + break; + + case gcvFEATURE_MMU: +#if gcdCAPTURE_ONLY_MODE + available = gcvTRUE; +#else + available = database->REG_MMU; +#endif + + break; + + case gcvFEATURE_FENCE_64BIT: + available = database->FENCE_64BIT; + break; + + case gcvFEATURE_TEX_BASELOD: + available = database->REG_Halti2; + + if (_IsHardwareMatch(Hardware, gcv900, 0x5250)) + { + available = gcvTRUE; + } + break; + + case gcvFEATURE_TEX_CACHE_FLUSH_FIX: + available = database->REG_Halti5; + break; + + case gcvFEATURE_BUG_FIXES1: + available = database->REG_BugFixes1; + break; + + case gcvFEATURE_MULTI_SOURCE_BLT: + available = database->REG_MultiSourceBlt; + break; + + case gcvFEATURE_HALTI5: + available = database->REG_Halti5; + break; + + case gcvFEATURE_FAST_CLEAR: + available = database->REG_FastClear; + + if (Hardware->identity.chipModel == gcv700) + { + available = gcvFALSE; + } + break; + + case gcvFEATURE_BUG_FIXES7: + available = database->REG_BugFixes7; + break; + + case gcvFEATURE_ZCOMPRESSION: + available = database->REG_ZCompression; + break; + + case gcvFEATURE_SHADER_HAS_INSTRUCTION_CACHE: + available = database->REG_InstructionCache; + break; + + case gcvFEATURE_YUV420_TILER: + available = database->REG_YUV420Tiler; + break; + + case gcvFEATURE_2DPE20: + available = database->REG_2DPE20; + break; + + case gcvFEATURE_DITHER_AND_FILTER_PLUS_ALPHA_2D: + available = database->REG_DitherAndFilterPlusAlpha2D; + break; + + case gcvFEATURE_ONE_PASS_2D_FILTER: + available = database->REG_OnePass2DFilter; + break; + + case gcvFEATURE_HALTI1: + available = database->REG_Halti1; + break; + + case gcvFEATURE_HALTI3: + available = database->REG_Halti3; + break; + + case gcvFEATURE_HALTI4: + available = database->REG_Halti4; + break; + + case gcvFEATURE_GEOMETRY_SHADER: + available = database->REG_GeometryShader; + break; + + case gcvFEATURE_TESSELLATION: + available = database->REG_TessellationShaders; + break; + + case gcvFEATURE_GENERIC_ATTRIB: + available = database->REG_Generics; + break; + + case gcvFEATURE_TEXTURE_LINEAR: + available = database->REG_LinearTextureSupport; + break; + + case gcvFEATURE_TX_FILTER: + available = database->REG_TXFilter; + break; + + case gcvFEATURE_TX_SUPPORT_DEC: + available = database->REG_TXSupportDEC; + break; + + case gcvFEATURE_TX_FRAC_PRECISION_6BIT: + available = database->REG_TX6bitFrac; + break; + + case gcvFEATURE_TEXTURE_ASTC: + available = database->REG_TXEnhancements4 && !database->NO_ASTC; + break; + + case gcvFEATURE_SHADER_ENHANCEMENTS2: + available = database->REG_SHEnhancements2; + break; + + case gcvFEATURE_BUG_FIXES18: + available = database->REG_BugFixes18; + break; + + case gcvFEATURE_64K_L2_CACHE: + available = gcvFALSE; + break; + + case gcvFEATURE_BUG_FIXES4: + available = database->REG_BugFixes4; + break; + + case gcvFEATURE_BUG_FIXES12: + available = database->REG_BugFixes12; + break; + + case gcvFEATURE_HW_TFB: + available = database->HWTFB; + break; + + case gcvFEATURE_SNAPPAGE_CMD_FIX: + available = database->SH_SNAP2PAGE_FIX; + break; + + case gcvFEATURE_SECURITY: + available = database->SECURITY; + break; + + case gcvFEATURE_TX_DESCRIPTOR: + available = database->REG_Halti5; + break; + + case gcvFEATURE_TX_DESC_CACHE_CLOCKGATE_FIX: + available = database->TX_DESC_CACHE_CLOCKGATE_FIX; + break; + + case gcvFEATURE_ROBUSTNESS: + available = database->ROBUSTNESS; + break; + + case gcvFEATURE_SNAPPAGE_CMD: + available = database->SNAPPAGE_CMD; + break; + + case gcvFEATURE_HALF_FLOAT_PIPE: + available = database->REG_HalfFloatPipe; + break; + + case gcvFEATURE_SH_INSTRUCTION_PREFETCH: + available = database->SH_ICACHE_PREFETCH; + break; + + case gcvFEATURE_FE_NEED_DUMMYDRAW: + available = database->FE_NEED_DUMMYDRAW; + + if (_IsHardwareMatch(Hardware, gcv600, 0x4653)) + { + available = gcvTRUE; + } + + break; + + case gcvFEATURE_DEC300_COMPRESSION: + available = database->REG_DEC; + break; + + case gcvFEATURE_DEC400_COMPRESSION: + available = database->G2D_DEC400; + break; + + case gcvFEATURE_DEC400EX_COMPRESSION: + available = database->G2D_DEC400EX; + break; + + case gcvFEATURE_TPC_COMPRESSION: + available = database->REG_ThirdPartyCompression; + break; + + case gcvFEATURE_TPCV11_COMPRESSION: + available = database->G2D_3rd_PARTY_COMPRESSION_1_1; + break; + + case gcvFEATURE_USC_DEFER_FILL_FIX: + available = database->USC_DEFER_FILL_FIX; + break; + + case gcvFEATURE_USC: + available = database->REG_Halti5; + break; + + case gcvFEATURE_RA_CG_FIX: + available = database->RA_CG_FIX; + break; + + case gcvFEATURE_MULTI_CLUSTER: + available = database->MULTI_CLUSTER; + break; + + case gcvFEATURE_ZERO_ATTRIB_SUPPORT: + available = database->REG_Halti4; + break; + + case gcvFEATURE_SH_CLOCK_GATE_FIX: + available = database->SH_CLOCK_GATE_FIX; + break; + + case gcvFEATURE_GPIPE_CLOCK_GATE_FIX: + available = gcvFALSE; + break; + + case gcvFEATURE_NEW_GPIPE: + available = database->NEW_GPIPE; + break; + + case gcvFEATURE_MULTI_CORE_BLOCK_SET_CONFIG2: + available = database->MULTI_CORE_BLOCK_SET_CONFIG2; + break; + + case gcvFEATURE_SECURITY_AHB: + available = database->SECURITY_AHB; + break; + + case gcvFEATURE_SMALL_BATCH: + available = database->SMALLBATCH; + break; + + case gcvFEATURE_ASYNC_BLIT: + available = database->ASYNC_BLT; + break; + + case gcvFEATURE_PSCS_THROTTLE: + available = database->PSCS_THROTTLE; + break; + + case gcvFEATURE_SEPARATE_LS: + available = database->SEPARATE_LS; + break; + + case gcvFEATURE_MCFE: + available = database->MCFE; + break; + + case gcvFEATURE_COMPUTE_ONLY: + available = database->COMPUTE_ONLY; + break; + + case gcvFEATURE_USC_FULLCACHE_FIX: + available = database->USC_FULL_CACHE_FIX; + break; + + case gcvFEATURE_PE_TILE_CACHE_FLUSH_FIX: + available = database->PE_TILE_CACHE_FLUSH_FIX; + break; + + case gcvFEATURE_TILE_STATUS_2BITS: + available = database->REG_TileStatus2Bits; + break; + + case gcvFEATURE_128BTILE: + available = database->CACHE128B256BPERLINE; + break; + + case gcvFEATURE_COMPRESSION_DEC400: + available = database->DEC400; + break; + + case gcvFEATURE_SUPPORT_GCREGTX: + available = database->REG_Halti1; + + if (Hardware->identity.chipModel == gcv880 && + ((Hardware->identity.chipRevision & 0xfff0) == 0x5120)) + { + available = gcvTRUE; + } + break; + + case gcvFEATURE_MSAA_FRAGMENT_OPERATION: + available = database->MSAA_FRAGMENT_OPERATION; + break; + + case gcvFEATURE_OCB_COUNTER: + available = database->OCB_COUNTER; + break; + + case gcvFEATURE_AI_GPU: + available = database->AI_GPU; + break; + + case gcvFEATURE_NN_ENGINE: + available = database->NNCoreCount > 0; + break; + + case gcvFEATURE_TP_ENGINE: + available = database->TP_ENGINE; + break; + + case gcvFEATURE_HI_REORDER_FIX: + available = database->HI_REORDER_FIX; + break; + + case gcvFEATURE_EVIS2_FLOP_RESET_FIX: + available = database->EVIS2_FLOP_RESET_FIX; + break; + + case gcvFEATURE_USC_ASYNC_CP_RTN_FLOP_RESET_FIX: + available = database->USC_ASYNC_CP_RTN_FLOP_RESET_FIX; + break; + + case gcvFEATURE_USC_EVICT_CTRL_FIFO_FLOP_RESET_FIX: + available = database->USC_EVICT_CTRL_FIFO_FLOP_RESET_FIX; + /*FALLTHRU*/ + default: + gcmkFATAL("Invalid feature has been requested."); + available = gcvFALSE; + /*FALLTHRU*/ + } + + gcmkFOOTER_ARG("%d", available ? gcvSTATUS_TRUE : gcvSTATUS_FALSE); + return available; +} + +static void +_ConfigurePolicyID( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gctUINT32 policyID; + gctUINT32 auxBit = ~0U; + gctUINT32 axiConfig; + gckOS os = Hardware->os; + gceCORE core = Hardware->core; + gctUINT32 i; + gctUINT32 offset; + gctUINT32 shift; + gctUINT32 currentAxiConfig; + + status = gckOS_GetPolicyID(os, gcvVIDMEM_TYPE_GENERIC, &policyID, &axiConfig); + + if (status == gcvSTATUS_NOT_SUPPORTED) + { + /* No customized policyID setting. */ + return; + } + + for (i = 0; i < 16; i++) + { + /* Mapping 16 surface type.*/ + status = gckOS_GetPolicyID(os, (gceVIDMEM_TYPE) i, &policyID, &axiConfig); + + if (gcmIS_SUCCESS(status)) + { + if (auxBit == ~0U) + { + /* There is a customized policyID setting for this type. */ + auxBit = (policyID >> 4) & 0x1; + } + else + { + /* Check whether this bit changes. */ + if (auxBit != ((policyID >> 4) & 0x1)) + { + gcmkPRINT("[galcore]: AUX_BIT changes"); + return; + } + } + + offset = policyID >> 1; + + shift = (policyID & 0x1) * 16; + + axiConfig &= 0xFFFF; + + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + os, + core, + (0x0070 + offset) << 2, + ¤tAxiConfig + )); + + currentAxiConfig |= (axiConfig << shift); + + gcmkVERIFY_OK(gckOS_WriteRegisterEx( + os, + core, + (0x0070 + offset) << 2, + currentAxiConfig + )); + } + } + + if (auxBit != ~0U) + { + gcmkVERIFY_OK(gckOS_WriteRegisterEx( + os, + core, + 0x000EC, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:8) - (0 ? + 8:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:8) - (0 ? + 8:8) + 1))))))) << (0 ? + 8:8))) | (((gctUINT32) ((gctUINT32) (auxBit) & ((gctUINT32) ((((1 ? + 8:8) - (0 ? + 8:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 8:8) - (0 ? 8:8) + 1))))))) << (0 ? 8:8))) + )); + } +} +/**************************** +** Initialise hardware options +*/ +static void +_SetHardwareOptions( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gctUINT64 data = 0; + gcsHAL_QUERY_CHIP_OPTIONS *options = &Hardware->options; + gcsFEATURE_DATABASE *database = Hardware->featureDatabase; + gctBOOL featureUSC = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_USC) ? gcvTRUE : gcvFALSE; + gctBOOL featureSeparateLS = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_SEPARATE_LS) ? gcvTRUE : gcvFALSE; + gctBOOL featureComputeOnly = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_COMPUTE_ONLY) ? gcvTRUE : gcvFALSE; + gctBOOL featureTS = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_TESSELLATION) ? gcvTRUE : gcvFALSE; + gctUINT32 featureL1CacheSize = database->L1CacheSize; + gctUINT32 featureUSCMaxPages = database->USC_MAX_PAGES; + + + status = gckOS_QueryOption(Hardware->os, "powerManagement", &data); + options->powerManagement = (data != 0); + + if (status == gcvSTATUS_NOT_SUPPORTED) + { + /* Enable power management by default. */ + options->powerManagement = gcvTRUE; + } + + /* Disable profiler by default */ + status = gckOS_QueryOption(Hardware->os, "gpuProfiler", &data); + options->gpuProfiler = (data != 0); + + if (status == gcvSTATUS_NOT_SUPPORTED) + { + /* Disable profiler by default */ + options->gpuProfiler= gcvFALSE; + } + + status = gckOS_QueryOption(Hardware->os, "mmu", &data); + options->enableMMU = (data != 0); + + if (status == gcvSTATUS_NOT_SUPPORTED) + { + /* Disable MMU if we can't get result from OS layer query */ + options->enableMMU = gcvFALSE; + } + + gcmCONFIGUSC2(gcmk, featureUSC, featureSeparateLS, featureComputeOnly, featureTS, + featureL1CacheSize, featureUSCMaxPages, + Hardware->options.uscAttribCacheRatio, Hardware->options.uscL1CacheRatio); + + status = gckOS_QueryOption(Hardware->os, "smallBatch", &data); + options->smallBatch = (data != 0); + + if (status == gcvSTATUS_NOT_SUPPORTED) + { + options->smallBatch = gcvTRUE; + } + + status = gckOS_QueryOption(Hardware->os, "userClusterMask", &data); + options->userClusterMask = (gctUINT32)data; + + if (status == gcvSTATUS_NOT_SUPPORTED || (options->userClusterMask == 0)) + { + /* use all clusters identified in database */ + options->userClusterMask = Hardware->identity.clusterAvailMask; + } + else if (options->userClusterMask & (~Hardware->identity.clusterAvailMask)) + { + gcmkPRINT("%s(%d): user cluster mask(0x%x) must be a subset of available clusters(0x%x),ignored it!", + __FUNCTION__, __LINE__, options->userClusterMask, Hardware->identity.clusterAvailMask); + options->userClusterMask= Hardware->identity.clusterAvailMask; + } + + options->secureMode = gcvSECURE_NONE; + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_SECURITY)) + { + gcmkASSERT(gcvSTATUS_TRUE == gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_SECURITY_AHB)); + + options->secureMode = gcvSECURE_IN_NORMAL; + + status = gckOS_QueryOption(Hardware->os, "TA", &data); + + if (gcmIS_SUCCESS(status) && data) + { + options->secureMode = gcvSECURE_IN_TA; + } + } + else if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_SECURITY_AHB)) + { + options->secureMode = gcvSECURE_IN_NORMAL; + } + + return; +} + +/* +* State timer helper must be called with powerMutex held. +*/ +static void +gckSTATETIMER_Reset( + IN gcsSTATETIMER * StateTimer, + IN gctUINT64 Start + ) +{ + gctUINT64 now; + + if (Start) + { + now = Start; + } + else + { + gckOS_GetProfileTick(&now); + } + + StateTimer->recent = StateTimer->start = now; + + gckOS_ZeroMemory(StateTimer->elapse, gcmSIZEOF(StateTimer->elapse)); +} + +gceSTATUS +gckHARDWARE_StartTimerReset( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gcmkHEADER(); + + gckSTATETIMER_Reset(&Hardware->powerStateCounter, 0); + + gcmkFOOTER(); + return status; +} + +static void +gckSTATETIMER_Accumulate( + IN gcsSTATETIMER * StateTimer, + IN gceCHIPPOWERSTATE OldState + ) +{ + gctUINT64 now; + gctUINT64 elapse; + + gckOS_GetProfileTick(&now); + + elapse = now - StateTimer->recent; + + StateTimer->recent = now; + + StateTimer->elapse[OldState] += elapse; +} + +static void +gckSTATETIMER_Query( + IN gcsSTATETIMER * StateTimer, + IN gceCHIPPOWERSTATE State, + OUT gctUINT64_PTR On, + OUT gctUINT64_PTR Off, + OUT gctUINT64_PTR Idle, + OUT gctUINT64_PTR Suspend + ) +{ + gckSTATETIMER_Accumulate(StateTimer, State); + + *On = StateTimer->elapse[gcvPOWER_ON]; + *Off = StateTimer->elapse[gcvPOWER_OFF]; + *Idle = StateTimer->elapse[gcvPOWER_IDLE]; + *Suspend = StateTimer->elapse[gcvPOWER_SUSPEND]; +} + +static gceSTATUS +_InitPageTableArray( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gcmkHEADER_ARG("Hardware=%p", Hardware); + + if (Hardware->options.secureMode == gcvSECURE_IN_NORMAL) + { + gcePOOL pool = gcvPOOL_DEFAULT; + gctUINT32 flags = gcvALLOC_FLAG_CONTIGUOUS; + +#if defined(CONFIG_ZONE_DMA32) || defined(CONFIG_ZONE_DMA) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,37) + flags |= gcvALLOC_FLAG_4GB_ADDR; +#endif +#endif + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + flags |= gcvALLOC_FLAG_CACHEABLE; +#endif + + Hardware->pagetableArray.size = 1024; + + /* Allocate mmu table array within 32bit space */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + Hardware->kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + flags, + &Hardware->pagetableArray.size, + &pool, + &Hardware->pagetableArray.videoMem + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + Hardware->kernel, + Hardware->pagetableArray.videoMem, + gcvFALSE, + gcvFALSE, + &Hardware->pagetableArray.logical + )); + + /* Get CPU physical address. */ + gcmkONERROR(gckVIDMEM_NODE_GetPhysical( + Hardware->kernel, + Hardware->pagetableArray.videoMem, + 0, + &Hardware->pagetableArray.address + )); + + /* Convert to GPU physical address. */ + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( + Hardware->os, + Hardware->pagetableArray.address, + &Hardware->pagetableArray.address + )); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (Hardware->pagetableArray.videoMem) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + Hardware->kernel, + Hardware->pagetableArray.videoMem + )); + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_SetupSRAMVidMem( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT i; + + for (i = gcvSRAM_INTERNAL0; i < gcvSRAM_INTER_COUNT; i++) + { + if (Hardware->identity.sRAMSizes[i] && + (Hardware->identity.sRAMBases[i] != gcvINVALID_PHYSICAL_ADDRESS)) + { + /* If the internal SRAM usage is memory block. */ + status = gckVIDMEM_Construct( + Hardware->os, + Hardware->identity.sRAMBases[i], + Hardware->identity.sRAMSizes[i], + 64, + 0, + &Hardware->sRAMVidMem[i] + ); + + if (gcmIS_ERROR(status)) + { + Hardware->identity.sRAMSizes[i] = 0; + Hardware->sRAMVidMem[i] = gcvNULL; + } + else + { + char sRAMName[20]; + gctUINT64 data = 0; + gctBOOL sRAMRequested; + + gcmkSPRINTF(sRAMName, gcmSIZEOF(sRAMName) - 1, "Galcore core%d sram%d", Hardware->core, i); + status = gckOS_QueryOption(Hardware->os, "sRAMRequested", (gctUINT64 *)&data); + sRAMRequested = (status == gcvSTATUS_OK) ? (data != 0) : gcvFALSE; + + gcmkONERROR(gckOS_RequestReservedMemory( + Hardware->os, + Hardware->identity.sRAMBases[i], + Hardware->identity.sRAMSizes[i], + sRAMName, + sRAMRequested, + &Hardware->sRAMPhysical[i] + )); + + Hardware->sRAMVidMem[i]->physical = Hardware->sRAMPhysical[i]; + } + } + } + +OnError: + return status; +} + +/******************************************************************************\ +****************************** gckHARDWARE API code ***************************** +\******************************************************************************/ + +/******************************************************************************* +** +** gckHARDWARE_Construct +** +** Construct a new gckHARDWARE object. +** +** INPUT: +** +** gckOS Os +** Pointer to an initialized gckOS object. +** +** gceCORE Core +** Specified core. +** +** OUTPUT: +** +** gckHARDWARE * Hardware +** Pointer to a variable that will hold the pointer to the gckHARDWARE +** object. +*/ +gceSTATUS +gckHARDWARE_Construct( + IN gckOS Os, + IN gckDEVICE Device, + IN gceCORE Core, + OUT gckHARDWARE * Hardware + ) +{ + gceSTATUS status; + gckHARDWARE hardware = gcvNULL; + gctUINT16 data = 0xff00; + gctPOINTER pointer = gcvNULL; + gctUINT i; + + gcmkHEADER_ARG("Os=0x%x", Os); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Os, gcvOBJ_OS); + gcmkVERIFY_ARGUMENT(Hardware != gcvNULL); + + /* Enable the GPU. */ + gcmkONERROR(gckOS_SetGPUPower(Os, Core, gcvTRUE, gcvTRUE)); + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x00000, + 0x00000900)); + + /* Allocate the gckHARDWARE object. */ + gcmkONERROR(gckOS_Allocate(Os, + gcmSIZEOF(struct _gckHARDWARE), + &pointer)); + + gckOS_ZeroMemory(pointer, gcmSIZEOF(struct _gckHARDWARE)); + + hardware = (gckHARDWARE) pointer; + + /* Initialize the gckHARDWARE object. */ + hardware->object.type = gcvOBJ_HARDWARE; + hardware->os = Os; + hardware->core = Core; + + gcmkONERROR(_GetHardwareSignature(hardware, Os, Core, &hardware->signature)); + + /* Identify the hardware. */ + gcmkONERROR(_IdentifyHardwareByDatabase(hardware, Os, Device, Core, &hardware->identity)); + + /* Setup SRAM memory heap. */ + gcmkONERROR(_SetupSRAMVidMem(hardware)); + + _SetHardwareOptions(hardware); + + hardware->mmuVersion = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_MMU); + + /* Get the system's physical base address for old MMU */ + if (hardware->mmuVersion == 0) + { + gcmkONERROR(gckOS_GetBaseAddress(Os, &hardware->baseAddress)); + } + + /* Determine the hardware type */ + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_PIPE_3D) + && gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_PIPE_2D) + ) + { + hardware->type = gcvHARDWARE_3D2D; + } + else if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_PIPE_2D)) + { + hardware->type = gcvHARDWARE_2D; + } + else if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_AI_GPU)) + { + hardware->type = gcvHARDWARE_3D; + } + else if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_NN_ENGINE) + || gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_TP_ENGINE) + ) + { + hardware->type = gcvHARDWARE_VIP; + } + else + { + hardware->type = gcvHARDWARE_3D; + } + + hardware->powerBaseAddress + = ((hardware->identity.chipModel == gcv300) + && (hardware->identity.chipRevision < 0x2000)) + ? 0x0100 + : 0x0000; + + + /* _ResetGPU need powerBaseAddress. */ + status = _ResetGPU(hardware, Os, Core); + if (status != gcvSTATUS_OK) + { + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, "_ResetGPU failed: status=%d\n", status); + } + +#if gcdDEC_ENABLE_AHB + gcmkONERROR(gckOS_WriteRegisterEx(Os, gcvCORE_DEC, 0x18180, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 22:22) - (0 ? + 22:22) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 22:22) - (0 ? + 22:22) + 1))))))) << (0 ? + 22:22))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 22:22) - (0 ? + 22:22) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 22:22) - (0 ? 22:22) + 1))))))) << (0 ? 22:22))))); +#endif + + hardware->hasL2Cache = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_64K_L2_CACHE); + + if (!hardware->hasL2Cache) + { + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x0055C, + 0x00FFFFFF)); + } + + hardware->powerMutex = gcvNULL; + + /* Determine whether bug fixes #1 are present. */ + hardware->extraEventStates = (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_BUG_FIXES1) == gcvFALSE); + + /* Check if big endian */ + hardware->bigEndian = (*(gctUINT8 *)&data == 0xff); + + /* Initialize the fast clear. */ + gcmkONERROR(gckHARDWARE_SetFastClear(hardware, -1, -1)); + +#if !gcdENABLE_128B_MERGE + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_MULTI_SOURCE_BLT)) + { + /* 128B merge is turned on by default. Disable it. */ + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x00558, 0)); + } + +#endif + +#if (gcdFPGA_BUILD && 1) + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_TPCV11_COMPRESSION)) + { + gctUINT32 data; + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00558, &data)); + data |= 0x1 << 27; + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x00558, data)); + } +#endif + + { + gctUINT32 value; + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x00090, &value)); +#if gcdDEC_ENABLE_AHB + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_DEC300_COMPRESSION)) + { + value |= ~0xFFFFFFBF; + } + else +#endif + { + value &= 0xFFFFFFBF; + } + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x00090, value)); + } + + /* Set power state to ON. */ + hardware->chipPowerState = gcvPOWER_ON; + hardware->clockState = gcvTRUE; + hardware->powerState = gcvTRUE; + hardware->lastWaitLink = ~0U; + hardware->lastEnd = ~0U; + hardware->globalSemaphore = gcvNULL; +#if gcdENABLE_FSCALE_VAL_ADJUST + hardware->powerOnFscaleVal = 64; +#endif + + gcmkONERROR(gckOS_CreateMutex(Os, &hardware->powerMutex)); + gcmkONERROR(gckOS_CreateSemaphore(Os, &hardware->globalSemaphore)); + + gcmkVERIFY_OK(gckOS_CreateTimer(Os, + _PowerStateTimerFunc, + (gctPOINTER)hardware, + &hardware->powerStateTimer)); + + for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) + { + gcmkONERROR(gckOS_AtomConstruct(Os, &hardware->pageTableDirty[i])); + } + + gcmkONERROR(gckOS_AtomConstruct(Os, &hardware->pendingEvent)); + + +#if defined(LINUX) || defined(__QNXNTO__) || defined(UNDER_CE) + if (hardware->mmuVersion) + { + hardware->stallFEPrefetch + = gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_ALLOW_STALL_PREFETCH_ENG); + } + else +#endif + { + hardware->stallFEPrefetch = + _QueryFeatureDatabase(hardware, gcvFEATURE_MCFE) == gcvFALSE; + } + + hardware->minFscaleValue = 1; + hardware->waitCount = 200; + +#if gcdLINK_QUEUE_SIZE + gcmkONERROR(gckQUEUE_Allocate(hardware->os, &hardware->linkQueue, gcdLINK_QUEUE_SIZE)); +#endif + + /* Initialize FEs, either MCFE or wait-link FE. */ + if (_QueryFeatureDatabase(hardware, gcvFEATURE_MCFE)) + { + hardware->mcfeChannels[0] = gcvMCFE_CHANNEL_SYSTEM; + hardware->mcfeChannels[1] = gcvMCFE_CHANNEL_SHADER; + hardware->mcfeChannels[2] = gcvMCFE_CHANNEL_NN; + hardware->mcfeChannels[3] = gcvMCFE_CHANNEL_TP; + + hardware->mcfeChannelCount = 4; + + gcmkONERROR(gckMCFE_Construct(hardware, &hardware->mcFE)); + } + else + { + gcmkONERROR(gckWLFE_Construct(hardware, &hardware->wlFE)); + } + + if (_QueryFeatureDatabase(hardware, gcvFEATURE_ASYNC_BLIT)) + { + gcmkONERROR(gckASYNC_FE_Construct(hardware, &hardware->asyncFE)); + } + + /* Construct hardware function */ + gcmkONERROR(gckFUNCTION_Construct(hardware)); + + /* Return pointer to the gckHARDWARE object. */ + *Hardware = hardware; + + /* Success. */ + gcmkFOOTER_ARG("*Hardware=0x%x", *Hardware); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (hardware != gcvNULL) + { + /* Turn off the power. */ + gcmkVERIFY_OK(gckOS_SetGPUPower(Os, Core, gcvFALSE, gcvFALSE)); + + if (hardware->globalSemaphore != gcvNULL) + { + /* Destroy the global semaphore. */ + gcmkVERIFY_OK(gckOS_DestroySemaphore(Os, + hardware->globalSemaphore)); + } + + if (hardware->powerMutex != gcvNULL) + { + /* Destroy the power mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Os, hardware->powerMutex)); + } + + if (hardware->powerStateTimer != gcvNULL) + { + gcmkVERIFY_OK(gckOS_StopTimer(Os, hardware->powerStateTimer)); + gcmkVERIFY_OK(gckOS_DestroyTimer(Os, hardware->powerStateTimer)); + } + + for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) + { + if (hardware->pageTableDirty[i] != gcvNULL) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(Os, hardware->pageTableDirty[i])); + } + } + + if (hardware->pendingEvent != gcvNULL) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(Os, hardware->pendingEvent)); + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, hardware)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_PostConstruct( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gctUINT i; + + /* Initialize MMU page table array. */ + gcmkONERROR(_InitPageTableArray(Hardware)); + + for (i = 0; i < gcvFUNCTION_EXECUTION_NUM; i++) + { + gctBOOL funcVaild = gcvFALSE; + + gckFUNCTION_Validate(&Hardware->functions[i], &funcVaild); + if (funcVaild) + { + gcmkONERROR(gckFUNCTION_Init(&Hardware->functions[i])); + } + } + + return gcvSTATUS_OK; + +OnError: + for (i = 0; i < gcvFUNCTION_EXECUTION_NUM; i++) + { + gckFUNCTION_Release(&Hardware->functions[i]); + } + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_PreDestroy +** +** Prepare destroying an gckHARDWARE object. +** This is to destroy resources relevant to other modules such as MMU. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to the gckHARDWARE object that needs to be destroyed. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_PreDestroy( + IN gckHARDWARE Hardware + ) +{ + gcmkHEADER_ARG("%x", Hardware); + + gcmkVERIFY_OK(gckFUNCTION_Destory(Hardware)); + + if (Hardware->pagetableArray.videoMem) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + Hardware->kernel, + Hardware->pagetableArray.videoMem, + 0, + gcvFALSE, + gcvFALSE + )); + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + Hardware->kernel, + Hardware->pagetableArray.videoMem + )); + + Hardware->pagetableArray.videoMem = gcvNULL; + Hardware->pagetableArray.logical = gcvNULL; + } + + if (Hardware->wlFE) + { + gckWLFE_Destroy(Hardware, Hardware->wlFE); + Hardware->wlFE = gcvNULL; + } + + if (Hardware->asyncFE) + { + gckASYNC_FE_Destroy(Hardware, Hardware->asyncFE); + Hardware->asyncFE = gcvNULL; + } + + if (Hardware->mcFE) + { + gckMCFE_Destroy(Hardware, Hardware->mcFE); + Hardware->mcFE = gcvNULL; + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_Destroy +** +** Destroy an gckHARDWARE object. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to the gckHARDWARE object that needs to be destroyed. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_Destroy( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gctUINT i; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + /* Destroy the power semaphore. */ + gcmkVERIFY_OK(gckOS_DestroySemaphore(Hardware->os, + Hardware->globalSemaphore)); + + /* Destroy the power mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Hardware->os, Hardware->powerMutex)); + + gcmkVERIFY_OK(gckOS_StopTimer(Hardware->os, Hardware->powerStateTimer)); + gcmkVERIFY_OK(gckOS_DestroyTimer(Hardware->os, Hardware->powerStateTimer)); + + for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(Hardware->os, Hardware->pageTableDirty[i])); + } + + gcmkVERIFY_OK(gckOS_AtomDestroy(Hardware->os, Hardware->pendingEvent)); + +#if gcdLINK_QUEUE_SIZE + gckQUEUE_Free(Hardware->os, &Hardware->linkQueue); +#endif + + /* Mark the object as unknown. */ + Hardware->object.type = gcvOBJ_UNKNOWN; + + /* Free the object. */ + gcmkONERROR(gcmkOS_SAFE_FREE(Hardware->os, Hardware)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_GetType +** +** Get the hardware type. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** OUTPUT: +** +** gceHARDWARE_TYPE * Type +** Pointer to a variable that receives the type of hardware object. +*/ +gceSTATUS +gckHARDWARE_GetType( + IN gckHARDWARE Hardware, + OUT gceHARDWARE_TYPE * Type + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + gcmkVERIFY_ARGUMENT(Type != gcvNULL); + + *Type = Hardware->type; + + gcmkFOOTER_ARG("*Type=%d", *Type); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_InitializeHardware +** +** Initialize the hardware. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to the gckHARDWARE object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_InitializeHardware( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gctUINT32 control; + gctUINT32 data; + gctUINT32 regPMC = 0; + gctUINT32 i; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + /* Disable isolate GPU bit. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + ((((gctUINT32) (0x00000900)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 19:19) - (0 ? + 19:19) + 1))))))) << (0 ? + 19:19))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 19:19) - (0 ? 19:19) + 1))))))) << (0 ? 19:19))))); + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + &control)); + + /* Enable debug register. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + ((((gctUINT32) (control)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 11:11) - (0 ? + 11:11) + 1))))))) << (0 ? + 11:11))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 11:11) - (0 ? 11:11) + 1))))))) << (0 ? 11:11))))); + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_SECURITY_AHB) && + (Hardware->options.secureMode == gcvSECURE_IN_NORMAL)) + { + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x003A8, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))))); + } + /* Reset memory counters. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0003C, + ~0U)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0003C, + 0)); + + if (Hardware->mmuVersion == 0) + { + /* Program the base addesses. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0041C, + Hardware->baseAddress)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00418, + Hardware->baseAddress)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00428, + Hardware->baseAddress)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00420, + Hardware->baseAddress)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00424, + Hardware->baseAddress)); + } + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00100, + &data)); + + /* Enable clock gating. */ + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + if ((Hardware->identity.chipRevision == 0x4301) + || (Hardware->identity.chipRevision == 0x4302) + ) + { + /* Disable stall module level clock gating for 4.3.0.1 and 4.3.0.2 + ** revisions. */ + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))); + } + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00100, + data)); + + /* Initialize FE. */ + if (Hardware->wlFE) + { + gckWLFE_Initialize(Hardware, Hardware->wlFE); + } + else if (Hardware->mcFE) + { + gckMCFE_Initialize(Hardware, gcvFALSE, Hardware->mcFE); + } + + if (Hardware->asyncFE) + { + gckASYNC_FE_Initialize(Hardware, Hardware->asyncFE); + } + +#if gcdENABLE_3D + /* Disable PE clock gating on revs < 5.0 when HZ is present without a + ** bug fix. */ + if ((Hardware->identity.chipRevision < 0x5000) + && gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_HZ) + && !gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_BUG_FIXES4) + ) + { + if (regPMC == 0) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + ®PMC)); + } + + /* Disable PE clock gating. */ + regPMC = ((((gctUINT32) (regPMC)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2))); + } +#endif + + if (Hardware->identity.chipModel == gcv4000 && + ((Hardware->identity.chipRevision == 0x5208) || (Hardware->identity.chipRevision == 0x5222))) + { + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + ((((gctUINT32) (0x01590880)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:23) - (0 ? + 23:23) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:23) - (0 ? + 23:23) + 1))))))) << (0 ? + 23:23))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 23:23) - (0 ? + 23:23) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:23) - (0 ? 23:23) + 1))))))) << (0 ? 23:23))))); + } + + if ((Hardware->identity.chipModel == gcv1000 && + (Hardware->identity.chipRevision == 0x5039 || + Hardware->identity.chipRevision == 0x5040)) + || + (Hardware->identity.chipModel == gcv2000 && + Hardware->identity.chipRevision == 0x5140) + ) + { + gctUINT32 pulseEater; + + pulseEater = ((((gctUINT32) (0x01590880)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))); + + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + ((((gctUINT32) (pulseEater)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:17) - (0 ? + 17:17) + 1))))))) << (0 ? + 17:17))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:17) - (0 ? 17:17) + 1))))))) << (0 ? 17:17))))); + } + + if ((gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_HALTI2) == gcvSTATUS_FALSE) + || (Hardware->identity.chipRevision < 0x5422) + ) + { + if (regPMC == 0) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + ®PMC)); + } + + regPMC = ((((gctUINT32) (regPMC)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:15) - (0 ? + 15:15) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:15) - (0 ? + 15:15) + 1))))))) << (0 ? + 15:15))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 15:15) - (0 ? + 15:15) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:15) - (0 ? 15:15) + 1))))))) << (0 ? 15:15))); + } + + if (_IsHardwareMatch(Hardware, gcv2000, 0x5108)) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00480, + &data)); + + /* Set FE bus to one, TX bus to zero */ + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))); + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7))); + + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00480, + data)); + } + + /* AHBDEC400 */ + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_DEC400EX_COMPRESSION)) + { + data = 0x0201018A; + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))); + gcmkONERROR(gckOS_WriteRegisterEx( + Hardware->os, + Hardware->core, + 0x00800, + data)); + + data = 0x003FC810; + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:0) - (0 ? + 6:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:0) - (0 ? + 6:0) + 1))))))) << (0 ? + 6:0))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 6:0) - (0 ? + 6:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:0) - (0 ? 6:0) + 1))))))) << (0 ? 6:0))); + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 13:7) - (0 ? + 13:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 13:7) - (0 ? + 13:7) + 1))))))) << (0 ? + 13:7))) | (((gctUINT32) ((gctUINT32) (2) & ((gctUINT32) ((((1 ? + 13:7) - (0 ? + 13:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 13:7) - (0 ? 13:7) + 1))))))) << (0 ? 13:7))); + gcmkONERROR(gckOS_WriteRegisterEx( + Hardware->os, + Hardware->core, + 0x00808, + data)); + } + +#if !gcdCAPTURE_ONLY_MODE + gcmkONERROR( + gckHARDWARE_SetMMU(Hardware, + Hardware->kernel->mmu)); +#endif + + if (Hardware->mcFE) + { + /* Reinitialize MCFE, now MMU is enabled. */ + gckMCFE_Initialize(Hardware, gcvTRUE, Hardware->mcFE); + } + + if (Hardware->identity.chipModel >= gcv400 + && Hardware->identity.chipModel != gcv420 + && !gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_BUG_FIXES12)) + { + if (regPMC == 0) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + ®PMC)); + } + + /* Disable PA clock gating. */ + regPMC = ((((gctUINT32) (regPMC)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))); + } + + /* Limit 2D outstanding request. */ + if (Hardware->maxOutstandingReads) + { + gctUINT32 data; + + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00414, + &data)); + + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (Hardware->maxOutstandingReads & 0xFF) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))); + + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00414, + data)); + } + + if (_IsHardwareMatch(Hardware, gcv1000, 0x5035)) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00414, + &data)); + + /* Disable HZ-L2. */ + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:12) - (0 ? + 12:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:12) - (0 ? + 12:12) + 1))))))) << (0 ? + 12:12))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 12:12) - (0 ? + 12:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:12) - (0 ? 12:12) + 1))))))) << (0 ? 12:12))); + + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00414, + data)); + } + + if (_IsHardwareMatch(Hardware, gcv4000, 0x5222) + || _IsHardwareMatch(Hardware, gcv2000, 0x5108) + || _IsHardwareMatch(Hardware, gcv7000, 0x6202) + || _IsHardwareMatch(Hardware, gcv7000, 0x6203) + || (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_TX_DESCRIPTOR) + && !gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_TX_DESC_CACHE_CLOCKGATE_FIX) + ) + ) + { + if (regPMC == 0) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + ®PMC)); + } + + /* Disable TX clock gating. */ + regPMC = ((((gctUINT32) (regPMC)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7))); + } + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_NEW_GPIPE) && + !gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_GPIPE_CLOCK_GATE_FIX)) + { + if (regPMC == 0) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + ®PMC)); + } + + /* Disable GPIPE clock gating. */ + regPMC = ((((gctUINT32) (regPMC)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 22:22) - (0 ? + 22:22) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 22:22) - (0 ? + 22:22) + 1))))))) << (0 ? + 22:22))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 22:22) - (0 ? + 22:22) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 22:22) - (0 ? 22:22) + 1))))))) << (0 ? 22:22))); + } + + if (_IsHardwareMatch(Hardware, gcv880, 0x5106)) + { + Hardware->kernel->timeOut = 140 * 1000; + } + + if (regPMC == 0) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + ®PMC)); + } + + /* Disable RA HZ clock gating. */ + regPMC = ((((gctUINT32) (regPMC)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:17) - (0 ? + 17:17) + 1))))))) << (0 ? + 17:17))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:17) - (0 ? 17:17) + 1))))))) << (0 ? 17:17))); + + /* Disable RA EZ clock gating. */ + regPMC = ((((gctUINT32) (regPMC)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))); + + if ((gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_HALTI5) + && !gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_RA_CG_FIX) + ) + ) + { + if (regPMC == 0) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + ®PMC)); + } + + /* Disable RA clock gating. */ + regPMC = ((((gctUINT32) (regPMC)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:6) - (0 ? + 6:6) + 1))))))) << (0 ? + 6:6))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:6) - (0 ? 6:6) + 1))))))) << (0 ? 6:6))); + } + + if ((gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_HALTI5) + && !gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_SH_CLOCK_GATE_FIX) + ) + ) + { + if (regPMC == 0) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + ®PMC)); + } + + /* Disable SH clock gating. */ + regPMC = ((((gctUINT32) (regPMC)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))); + } + + if (regPMC != 0) + { + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + regPMC)); + } + + if (_IsHardwareMatch(Hardware, gcv2000, 0x5108) + || (_IsHardwareMatch(Hardware, gcv2000, 0xffff5450)) + || _IsHardwareMatch(Hardware, gcv320, 0x5007) + || _IsHardwareMatch(Hardware, gcv320, 0x5303) + || _IsHardwareMatch(Hardware, gcv880, 0x5106) + || _IsHardwareMatch(Hardware, gcv400, 0x4645) + ) + { + /* Update GPU AXI cache atttribute. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00008, + 0x00002200)); + } + + if ((Hardware->identity.chipRevision > 0x5420) + && gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_PIPE_3D)) + { + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + &data)); + + /* Disable internal DFS. */ + data = +#if gcdDVFS + ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 18:18) - (0 ? + 18:18) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 18:18) - (0 ? + 18:18) + 1))))))) << (0 ? + 18:18))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 18:18) - (0 ? + 18:18) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 18:18) - (0 ? 18:18) + 1))))))) << (0 ? 18:18))) | +#endif + ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))) | + ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:17) - (0 ? + 17:17) + 1))))))) << (0 ? + 17:17))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:17) - (0 ? 17:17) + 1))))))) << (0 ? 17:17))); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + data)); + } + + if (_IsHardwareMatch(Hardware, gcv2500, 0x5422)) + { + gcmkONERROR(gckOS_ReadRegisterEx( + Hardware->os, Hardware->core, 0x00090, &data)); + + /* AXI switch setup to SPLIT_TO64 mode */ + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:0) - (0 ? + 1:0) + 1))))))) << (0 ? + 1:0))) | (((gctUINT32) (0x2 & ((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:0) - (0 ? 1:0) + 1))))))) << (0 ? 1:0))); + + gcmkONERROR(gckOS_WriteRegisterEx( + Hardware->os, Hardware->core, 0x00090, data)); + } + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_NN_ENGINE) && + (!gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_HI_REORDER_FIX) || + (((gcsFEATURE_DATABASE *)Hardware->featureDatabase)->AXI_SRAM_SIZE == 0)) + ) + { + gcmkONERROR(gckOS_ReadRegisterEx( + Hardware->os, Hardware->core, 0x00090, &data)); + + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:6) - (0 ? + 6:6) + 1))))))) << (0 ? + 6:6))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:6) - (0 ? 6:6) + 1))))))) << (0 ? 6:6))); + + gcmkONERROR(gckOS_WriteRegisterEx( + Hardware->os, Hardware->core, 0x00090, data)); + } + + _ConfigurePolicyID(Hardware); + +#if gcdDEBUG_MODULE_CLOCK_GATING + _ConfigureModuleLevelClockGating(Hardware); +#endif + + /* Perfrom hardware functions */ + for (i = 0; i < gcvFUNCTION_EXECUTION_NUM; i++) + { + gctBOOL funcVaild = gcvFALSE; + + /* Skip functions since it will perform at special place */ + if (i == gcvFUNCTION_EXECUTION_MMU || i == gcvFUNCTION_EXECUTION_FLUSH) + { + continue; + } + + gckFUNCTION_Validate(&Hardware->functions[i], &funcVaild); + if (funcVaild) + { + gckFUNCTION_Execute(&Hardware->functions[i]); + gckFUNCTION_Release(&Hardware->functions[i]); + } + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the error. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_QueryMemory +** +** Query the amount of memory available on the hardware. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to the gckHARDWARE object. +** +** OUTPUT: +** +** gctSIZE_T * InternalSize +** Pointer to a variable that will hold the size of the internal video +** memory in bytes. If 'InternalSize' is gcvNULL, no information of the +** internal memory will be returned. +** +** gctUINT32 * InternalBaseAddress +** Pointer to a variable that will hold the hardware's base address for +** the internal video memory. This pointer cannot be gcvNULL if +** 'InternalSize' is also non-gcvNULL. +** +** gctUINT32 * InternalAlignment +** Pointer to a variable that will hold the hardware's base address for +** the internal video memory. This pointer cannot be gcvNULL if +** 'InternalSize' is also non-gcvNULL. +** +** gctSIZE_T * ExternalSize +** Pointer to a variable that will hold the size of the external video +** memory in bytes. If 'ExternalSize' is gcvNULL, no information of the +** external memory will be returned. +** +** gctUINT32 * ExternalBaseAddress +** Pointer to a variable that will hold the hardware's base address for +** the external video memory. This pointer cannot be gcvNULL if +** 'ExternalSize' is also non-gcvNULL. +** +** gctUINT32 * ExternalAlignment +** Pointer to a variable that will hold the hardware's base address for +** the external video memory. This pointer cannot be gcvNULL if +** 'ExternalSize' is also non-gcvNULL. +** +** gctUINT32 * HorizontalTileSize +** Number of horizontal pixels per tile. If 'HorizontalTileSize' is +** gcvNULL, no horizontal pixel per tile will be returned. +** +** gctUINT32 * VerticalTileSize +** Number of vertical pixels per tile. If 'VerticalTileSize' is +** gcvNULL, no vertical pixel per tile will be returned. +*/ +gceSTATUS +gckHARDWARE_QueryMemory( + IN gckHARDWARE Hardware, + OUT gctSIZE_T * InternalSize, + OUT gctUINT32 * InternalBaseAddress, + OUT gctUINT32 * InternalAlignment, + OUT gctSIZE_T * ExternalSize, + OUT gctUINT32 * ExternalBaseAddress, + OUT gctUINT32 * ExternalAlignment, + OUT gctUINT32 * HorizontalTileSize, + OUT gctUINT32 * VerticalTileSize + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + if (InternalSize != gcvNULL) + { + /* No internal memory. */ + *InternalSize = 0; + } + + if (ExternalSize != gcvNULL) + { + /* No external memory. */ + *ExternalSize = 0; + } + + if (HorizontalTileSize != gcvNULL) + { + /* 4x4 tiles. */ + *HorizontalTileSize = 4; + } + + if (VerticalTileSize != gcvNULL) + { + /* 4x4 tiles. */ + *VerticalTileSize = 4; + } + + /* Success. */ + gcmkFOOTER_ARG("*InternalSize=%lu *InternalBaseAddress=0x%08x " + "*InternalAlignment=0x%08x *ExternalSize=%lu " + "*ExternalBaseAddress=0x%08x *ExtenalAlignment=0x%08x " + "*HorizontalTileSize=%u *VerticalTileSize=%u", + gcmOPT_VALUE(InternalSize), + gcmOPT_VALUE(InternalBaseAddress), + gcmOPT_VALUE(InternalAlignment), + gcmOPT_VALUE(ExternalSize), + gcmOPT_VALUE(ExternalBaseAddress), + gcmOPT_VALUE(ExternalAlignment), + gcmOPT_VALUE(HorizontalTileSize), + gcmOPT_VALUE(VerticalTileSize)); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_QueryChipIdentity +** +** Query the identity of the hardware. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to the gckHARDWARE object. +** +** OUTPUT: +** +** gcsHAL_QUERY_CHIP_IDENTITY_PTR Identity +** Pointer to the identity structure. +** +*/ +gceSTATUS +gckHARDWARE_QueryChipIdentity( + IN gckHARDWARE Hardware, + OUT gcsHAL_QUERY_CHIP_IDENTITY_PTR Identity + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(Identity != gcvNULL); + + *Identity = Hardware->identity; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_QueryChipOptions +** +** Query the options of the hardware. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to the gckHARDWARE object. +** +** OUTPUT: +** +** gcsHAL_QUERY_CHIP_OPTIONS_PTR Options +** Pointer to the identity structure. +** +*/ +gceSTATUS +gckHARDWARE_QueryChipOptions( + IN gckHARDWARE Hardware, + OUT gcsHAL_QUERY_CHIP_OPTIONS_PTR Options + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(Options != gcvNULL); + + *Options = Hardware->options; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + + +/******************************************************************************* +** +** gckHARDWARE_SplitMemory +** +** Split a hardware specific memory address into a pool and offset. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to the gckHARDWARE object. +** +** gctUINT32 Address +** Address in hardware specific format. +** +** OUTPUT: +** +** gcePOOL * Pool +** Pointer to a variable that will hold the pool type for the address. +** +** gctUINT32 * Offset +** Pointer to a variable that will hold the offset for the address. +*/ +gceSTATUS +gckHARDWARE_SplitMemory( + IN gckHARDWARE Hardware, + IN gctUINT32 Address, + OUT gcePOOL * Pool, + OUT gctUINT32 * Offset + ) +{ + gcmkHEADER_ARG("Hardware=0x%x Addres=0x%08x", Hardware, Address); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(Pool != gcvNULL); + gcmkVERIFY_ARGUMENT(Offset != gcvNULL); + + if (Hardware->mmuVersion == 0) + { + /* Dispatch on memory type. */ + switch ((((((gctUINT32) (Address)) >> (0 ? 31:31)) & ((gctUINT32) ((((1 ? 31:31) - (0 ? 31:31) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 31:31) - (0 ? 31:31) + 1)))))) )) + { + case 0x0: + /* System memory. */ + *Pool = gcvPOOL_SYSTEM; + break; + + case 0x1: + /* Virtual memory. */ + *Pool = gcvPOOL_VIRTUAL; + break; + + default: + /* Invalid memory type. */ + gcmkFOOTER_ARG("status=%d", gcvSTATUS_INVALID_ARGUMENT); + return gcvSTATUS_INVALID_ARGUMENT; + } + + /* Return offset of address. */ + *Offset = (((((gctUINT32) (Address)) >> (0 ? 30:0)) & ((gctUINT32) ((((1 ? 30:0) - (0 ? 30:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 30:0) - (0 ? 30:0) + 1)))))) ); + } + else + { + *Pool = gcvPOOL_SYSTEM; + *Offset = Address; + } + + /* Success. */ + gcmkFOOTER_ARG("*Pool=%d *Offset=0x%08x", *Pool, *Offset); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_PipeSelect +** +** Append a PIPESELECT command at the specified user logical memory. +** +** WARNING: Only for writing to USERSPACE! +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** gctPOINTER Logical +** Pointer to the current location inside the command queue to append +** the PIPESELECT command at or gcvNULL just to query the size of the +** PIPESELECT command. +** +** gcePIPE_SELECT Pipe +** Pipe value to select. +** +** gctSIZE_T * Bytes +** Pointer to the number of bytes available for the PIPESELECT command. +** If 'Logical' is gcvNULL, this argument will be ignored. +** +** OUTPUT: +** +** gctSIZE_T * Bytes +** Pointer to a variable that will receive the number of bytes required +** for the PIPESELECT command. If 'Bytes' is gcvNULL, nothing will be +** returned. +*/ +gceSTATUS +gckHARDWARE_PipeSelect( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gcePIPE_SELECT Pipe, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x Pipe=%d *Bytes=%lu", + Hardware, Logical, Pipe, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + + /* Append a PipeSelect. */ + if (Logical != gcvNULL) + { + gctUINT32 flush, stall; + + if (*Bytes < 32) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + flush = (Pipe == gcvPIPE_2D) + ? + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + : ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))); + + stall = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* LoadState(AQFlush, 1), flush. */ + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + )); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + logical + 1, + flush + )); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%x: FLUSH 0x%x", logical, flush); + + /* LoadState(AQSempahore, 1), stall. */ + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + logical + 2, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + )); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + logical + 3, + stall + )); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%x: SEMAPHORE 0x%x", logical + 2, stall); + + /* Stall, stall. */ + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + logical + 4, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + )); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + logical + 5, + stall + )); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%x: STALL 0x%x", logical + 4, stall); + + /* LoadState(AQPipeSelect, 1), pipe. */ + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + logical + 6, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E00) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + )); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + logical + 7, + (Pipe == gcvPIPE_2D) + ? 0x1 + : 0x0 + )); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%x: PIPE %d", logical + 6, Pipe); + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the PIPESELECT command. */ + *Bytes = 32; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_FenceRender( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 FenceAddress, + IN gctUINT64 FenceData, + IN OUT gctUINT32 * Bytes + ) +{ + gckOS os = Hardware->os; + gctUINT32_PTR logical = (gctUINT32_PTR)Logical; + + gctUINT32 dataLow = (gctUINT32)FenceData; + gctUINT32 dataHigh = (gctUINT32)(FenceData >> 32); + + if (logical) + { + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E1A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + ); + + gcmkWRITE_MEMORY( + logical, + FenceAddress + ); + + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E26) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + ); + + gcmkWRITE_MEMORY( + logical, + dataHigh + ); + + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E1B) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + ); + + gcmkWRITE_MEMORY( + logical, + dataLow + ); + } + + if (Bytes) + { + *Bytes = gcdRENDER_FENCE_LENGTH; + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FenceBlt( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 FenceAddress, + IN gctUINT64 FenceData, + IN OUT gctUINT32 * Bytes + ) +{ + gckOS os = Hardware->os; + gctUINT32_PTR logical = (gctUINT32_PTR)Logical; + + gctUINT32 dataLow = (gctUINT32)FenceData; + gctUINT32 dataHigh = (gctUINT32)(FenceData >> 32); + + if (logical) + { + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + ); + + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + ); + + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x5029) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + ); + + gcmkWRITE_MEMORY( + logical, + FenceAddress + ); + + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502D) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + ); + + gcmkWRITE_MEMORY( + logical, + dataHigh + ); + + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + ); + + gcmkWRITE_MEMORY( + logical, + dataLow + ); + + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + ); + + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + ); + } + + if (Bytes) + { + *Bytes = gcdBLT_FENCE_LENGTH; + } + + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_Fence +** +** Append a HW FENCE states at the specified user logical memory. +** +** WARNING: Only for writing to USERSPACE! +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** gceENGINE Engine +** Engine type, render or 3d-blit currently. +** +** +** gctPOINTER Logical +** Pointer to the current location inside the command queue to append +** the PIPESELECT command at or gcvNULL just to query the size of the +** PIPESELECT command. +** +** gctUINT32 FenceAddress +** GPU address to write out the data. +** +** gctUINT64 FenceData +** The 64bit data to write out. +** +** gctSIZE_T * Bytes +** Pointer to the number of bytes available for the PIPESELECT command. +** If 'Logical' is gcvNULL, this argument will be ignored. +** +** OUTPUT: +** +** gctSIZE_T * Bytes +** Pointer to a variable that will receive the number of bytes required +** for the PIPESELECT command. If 'Bytes' is gcvNULL, nothing will be +** returned. +*/ +gceSTATUS +gckHARDWARE_Fence( + IN gckHARDWARE Hardware, + IN gceENGINE Engine, + IN gctPOINTER Logical, + IN gctUINT32 FenceAddress, + IN gctUINT64 FenceData, + IN OUT gctUINT32 * Bytes + ) +{ + if (Engine == gcvENGINE_RENDER) + { + return _FenceRender(Hardware, Logical, FenceAddress, FenceData, Bytes); + } + else + { + return _FenceBlt(Hardware, Logical, FenceAddress, FenceData, Bytes); + } +} + +/******************************************************************************* +** +** gckHARDWARE_UpdateQueueTail +** +** Update the tail of the command queue. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** gctPOINTER Logical +** Logical address of the start of the command queue. +** +** gctUINT32 Offset +** Offset into the command queue of the tail (last command). +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_UpdateQueueTail( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Offset + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x Offset=0x%08x", + Hardware, Logical, Offset); + + /* Verify the hardware. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + /* Force a barrier. */ + gcmkONERROR( + gckOS_MemoryBarrier(Hardware->os, Logical)); + + /* Notify gckKERNEL object of change. */ + gcmkONERROR( + gckKERNEL_Notify(Hardware->kernel, + gcvNOTIFY_COMMAND_QUEUE)); + + if (status == gcvSTATUS_CHIP_NOT_READY) + { + gcmkONERROR(gcvSTATUS_DEVICE); + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + + +static void +_ResumeWaitLinkFE( + gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gctUINT32 resume; + gctUINT32 bytes; + gctUINT32 idle; + + /* Make sure FE is idle. */ + do + { + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00004, + &idle)); + } + while (idle != 0x7FFFFFFF); + + gcmkDUMP(Hardware->os, "@[register.wait 0x%05X 0x%08X 0x%08X]", + 0x00004, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (~0U) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))), + idle); + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00664, + &resume)); + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00664, + &resume)); + + bytes = Hardware->hasL2Cache ? 24 : 16; + + /* Start Command Parser. */ + gckWLFE_AtomicExecute(Hardware, resume, bytes); + +OnError: + return; +} + +/******************************************************************************* +** +** gckHARDWARE_Interrupt +** +** Process an interrupt. This function will read the interrupt acknowledge +** register, stores the data, and return whether the interrupt is from us. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_Interrupt( + IN gckHARDWARE Hardware + ) +{ + gctUINT32 data = 0; + gctUINT32 dataEx = 0; + gceSTATUS status; + gceSTATUS statusEx; + + /* + * Notice: + * In isr here. + * We should return success when either FE or AsyncFE reports correct + * interrupts, so that isr can wake up threadRoutine for either FE. + * That means, only need return ERROR when both FEs reports ERROR. + */ + /* Read AQIntrAcknowledge register. */ + status = gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00010, + &data); + + if (gcmIS_ERROR(status)) + { + goto OnError; + } + + if (data == 0) + { + /* Not our interrupt. */ + status = gcvSTATUS_NOT_OUR_INTERRUPT; + } + else + { +#if gcdINTERRUPT_STATISTIC + gckOS_AtomClearMask(Hardware->pendingEvent, data); +#endif + + if (data & (1 << 29)) + { + /* Event ID 29 is not a normal event, but for invalidating pipe. */ + _ResumeWaitLinkFE(Hardware); + data &= ~(1 << 29); + } + + /* Inform gckEVENT of the interrupt. */ + status = gckEVENT_Interrupt(Hardware->kernel->eventObj, data); + } + + if (!Hardware->asyncFE) + { + /* Done. */ + goto OnError; + } + + /* Read BLT interrupt. */ + statusEx = gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x000D4, + &dataEx + ); + + if (gcmIS_ERROR(statusEx)) + { + /* + * Do not overwrite status here, so that former status from + * AQIntrAck is returned. + */ + goto OnError; + } + + /* + * This bit looks useless now, we can use this check if this interrupt is + * from FE. + */ + dataEx &= ~0x80000000; + + /* + * Descriptor fetched, update counter. + * We can't do this at dataEx != 0 only, because read HW acknowledge + * register will overwrite 0x007E4. If one + * interrupt we don't read it, we will miss it for ever. + */ + gckASYNC_FE_UpdateAvaiable(Hardware); + + /* Do not need report NOT_OUT_INTERRUPT error if dataEx is 0. */ + if (dataEx) + { + statusEx = gckEVENT_Interrupt(Hardware->kernel->asyncEvent, dataEx); + + if (gcmIS_SUCCESS(statusEx)) + { + /* At least AsyncFE is success, treat all as success. */ + status = gcvSTATUS_OK; + } + } + +OnError: + /* Return the status. */ + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_Notify +** +** This functions will handle the event notifications. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_Notify( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gcmkHEADER_ARG("Hardware=%p", Hardware); + + /* Handle events. */ + status = gckEVENT_Notify(Hardware->kernel->eventObj, 0); + + if (Hardware->asyncFE) + { + status = gckEVENT_Notify(Hardware->kernel->asyncEvent, 0); + } + + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_QueryCommandBuffer +** +** Query the command buffer alignment and number of reserved bytes. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** OUTPUT: +** +** gctSIZE_T * Alignment +** Pointer to a variable receiving the alignment for each command. +** +** gctSIZE_T * ReservedHead +** Pointer to a variable receiving the number of reserved bytes at the +** head of each command buffer. +** +** gctSIZE_T * ReservedTail +** Pointer to a variable receiving the number of bytes reserved at the +** tail of each command buffer. +*/ +gceSTATUS +gckHARDWARE_QueryCommandBuffer( + IN gckHARDWARE Hardware, + IN gceENGINE Engine, + OUT gctUINT32 * Alignment, + OUT gctUINT32 * ReservedHead, + OUT gctUINT32 * ReservedTail + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + if (Alignment != gcvNULL) + { + /* Align every 8 bytes. */ + *Alignment = 8; + } + + if (ReservedHead != gcvNULL) + { + /* Reserve space for SelectPipe(). */ + *ReservedHead = 32; + } + + if (ReservedTail != gcvNULL) + { + if (Engine == gcvENGINE_RENDER) + { + gcmkFOOTER_NO(); + return gcvSTATUS_NOT_SUPPORTED; + } + else + { + *ReservedTail = gcdBLT_FENCE_LENGTH; + } + } + + /* Success. */ + gcmkFOOTER_ARG("*Alignment=%lu *ReservedHead=%lu *ReservedTail=%lu", + gcmOPT_VALUE(Alignment), gcmOPT_VALUE(ReservedHead), + gcmOPT_VALUE(ReservedTail)); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_QuerySystemMemory +** +** Query the command buffer alignment and number of reserved bytes. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** OUTPUT: +** +** gctSIZE_T * SystemSize +** Pointer to a variable that receives the maximum size of the system +** memory. +** +** gctUINT32 * SystemBaseAddress +** Poinetr to a variable that receives the base address for system +** memory. +*/ +gceSTATUS +gckHARDWARE_QuerySystemMemory( + IN gckHARDWARE Hardware, + OUT gctSIZE_T * SystemSize, + OUT gctUINT32 * SystemBaseAddress + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + if (SystemSize != gcvNULL) + { + /* Maximum system memory can be 2GB. */ + *SystemSize = 1U << 31; + } + + if (SystemBaseAddress != gcvNULL) + { + /* Set system memory base address. */ + *SystemBaseAddress = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:31) - (0 ? + 31:31) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:31) - (0 ? + 31:31) + 1))))))) << (0 ? + 31:31))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 31:31) - (0 ? + 31:31) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:31) - (0 ? 31:31) + 1))))))) << (0 ? 31:31))); + } + + /* Success. */ + gcmkFOOTER_ARG("*SystemSize=%lu *SystemBaseAddress=%lu", + gcmOPT_VALUE(SystemSize), gcmOPT_VALUE(SystemBaseAddress)); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_SetMMU +** +** Set the page table base address. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** gckMMU Mmu +** Pointer to mmu object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_SetMMU( + IN gckHARDWARE Hardware, + IN gckMMU Mmu + ) +{ + gceSTATUS status; + gctUINT32 address = 0; + + gcmkHEADER_ARG("Hardware=0x%x Mmu=0x%x", Hardware, Mmu); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + if (Hardware->mmuVersion == 0) + { + /* mmu v0 only support 1 level translation, only uses stlb level mapping. */ + gcmkSAFECASTPHYSADDRT(address, Mmu->dynamicArea4K.stlbPhysical); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "Setting page table to 0x%08X", + address); + + /* Write the AQMemoryFePageTable register. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00400, + address)); + + /* Write the AQMemoryRaPageTable register. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00410, + address)); + + /* Write the AQMemoryTxPageTable register. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00404, + address)); + + + /* Write the AQMemoryPePageTable register. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00408, + address)); + + /* Write the AQMemoryPezPageTable register. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0040C, + address)); + } + else + { + gctBOOL mmuValid = gcvTRUE; + + gcmkONERROR(gckFUNCTION_Validate(&Hardware->functions[gcvFUNCTION_EXECUTION_MMU], &mmuValid)); + + if (mmuValid) + { + gctBOOL hwMmuDisabled = gcvTRUE; + + /* Force Disable MMU to guarantee setup command be read from physical addr */ + if (Hardware->options.secureMode == gcvSECURE_IN_NORMAL) + { + gctUINT32 regMmuCtrl = 0; + gcmkONERROR(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x00388, + ®MmuCtrl + )); + + hwMmuDisabled = ((((((gctUINT32) (regMmuCtrl)) >> (0 ? 0:0)) & ((gctUINT32) ((((1 ? 0:0) - (0 ? 0:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1)))))) ) == 0x1) + ? gcvFALSE + : gcvTRUE; + } + else + { + gctUINT32 regMmuCtrl = 0; + + gcmkONERROR(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x0018C, + ®MmuCtrl + )); + + hwMmuDisabled = ((((((gctUINT32) (regMmuCtrl)) >> (0 ? 0:0)) & ((gctUINT32) ((((1 ? 0:0) - (0 ? 0:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1)))))) ) == 0x1) + ? gcvFALSE + : gcvTRUE; + } + + if (hwMmuDisabled) + { + gcmkONERROR(gckFUNCTION_Execute(&Hardware->functions[gcvFUNCTION_EXECUTION_MMU])); + + /* Enable MMU. */ + if (Hardware->options.secureMode == gcvSECURE_IN_NORMAL) + { + gcmkONERROR(gckOS_WriteRegisterEx_NoDump( + Hardware->os, + Hardware->core, + 0x00388, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + )); + } + else + { + gcmkONERROR(gckOS_WriteRegisterEx( + Hardware->os, + Hardware->core, + 0x0018C, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (gcvTRUE) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + )); + } + } + } + + } + + /* Return the status. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_FlushMMU +** +** Flush the page table. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_FlushMMU( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN gctUINT32 SubsequentBytes, + IN OUT gctUINT32 * Bytes + ) +{ + gceSTATUS status; + gctUINT32_PTR buffer; + gctUINT32 flushSize; + gctBOOL bltEngine = gcvFALSE; + gctBOOL multiCluster = gcvFALSE; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + if (Hardware->mmuVersion == 0) + { + flushSize = 8; + } + else + { + flushSize = 10 * 4; + + bltEngine = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_BLT_ENGINE); + multiCluster = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_MULTI_CLUSTER); + + if (bltEngine) + { + flushSize += 4 * 4; + + if (multiCluster) + { + flushSize += 2 * 4; + } + } + } + + if (Logical) + { + if (*Bytes < flushSize) + { + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + /* Flush the memory controller. */ + if (Hardware->mmuVersion == 0) + { + buffer = (gctUINT32_PTR)Logical; + + buffer[0] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E04) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + buffer[1] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))); + } + else + { + gctUINT32 count; + gctUINT32 offset = 2; + gctUINT32 prefetchCount = 4; + gctUINT32 semaphore, stall; + + if (bltEngine) + { + prefetchCount += 2; + if (multiCluster) + { + prefetchCount++; + } + } + + buffer = (gctUINT32_PTR)Logical; + + count = SubsequentBytes >> 3; + + /* LINK to next slot to flush FE FIFO. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x08 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (prefetchCount) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ = Address + offset * gcmSIZEOF(gctUINT32); + + /* Flush MMU cache. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0061) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = (((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) & ((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7)))); + + if (bltEngine) + { + /* Blt lock. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + if (multiCluster) + { + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x50CE) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (Hardware->identity.clusterAvailMask & Hardware->options.userClusterMask) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))); + } + } + + /* Arm the PE-FE Semaphore. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + semaphore = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))); + + if (Hardware->stallFEPrefetch) + { + semaphore |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 29:28) - (0 ? + 29:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 29:28) - (0 ? + 29:28) + 1))))))) << (0 ? + 29:28))) | (((gctUINT32) (0x3 & ((gctUINT32) ((((1 ? + 29:28) - (0 ? + 29:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 29:28) - (0 ? 29:28) + 1))))))) << (0 ? 29:28))); + } + + if (bltEngine) + { + semaphore |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + else + { + semaphore |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + + *buffer++ = semaphore; + + /* STALL FE until PE is done flushing. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + stall = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))); + + if (Hardware->stallFEPrefetch) + { + stall |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 29:28) - (0 ? + 29:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 29:28) - (0 ? + 29:28) + 1))))))) << (0 ? + 29:28))) | (((gctUINT32) (0x3 & ((gctUINT32) ((((1 ? + 29:28) - (0 ? + 29:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 29:28) - (0 ? 29:28) + 1))))))) << (0 ? 29:28))); + } + + if (bltEngine) + { + stall |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + else + { + stall |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + + *buffer++ = stall; + + if (bltEngine) + { + /* Blt unlock. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + + /* LINK to next slot to flush FE FIFO. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x08 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (count) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *buffer++ = Address + flushSize; + } + } + + if (Bytes) + { + *Bytes = flushSize; + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_FlushAsyncMMU( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT32 semaphore, stall; + gctUINT32_PTR buffer; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x *Bytes=%lu", + Hardware, Logical, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + + if (Logical != gcvNULL) + { + buffer = (gctUINT32_PTR) Logical; + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + )); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer + 1, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + )); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer + 2, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0061) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + )); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer + 3, + (((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) & ((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7)))) + )); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer + 4, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + )); + + semaphore = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + if (Hardware->stallFEPrefetch) + { + semaphore |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 29:28) - (0 ? + 29:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 29:28) - (0 ? + 29:28) + 1))))))) << (0 ? + 29:28))) | (((gctUINT32) (0x3 & ((gctUINT32) ((((1 ? + 29:28) - (0 ? + 29:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 29:28) - (0 ? 29:28) + 1))))))) << (0 ? 29:28))); + } + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer + 5, + semaphore)); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer + 6, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + )); + + stall = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + if (Hardware->stallFEPrefetch) + { + stall |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 29:28) - (0 ? + 29:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 29:28) - (0 ? + 29:28) + 1))))))) << (0 ? + 29:28))) | (((gctUINT32) (0x3 & ((gctUINT32) ((((1 ? + 29:28) - (0 ? + 29:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 29:28) - (0 ? 29:28) + 1))))))) << (0 ? 29:28))); + } + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer + 7, + stall)); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer + 8, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + )); + + gcmkONERROR(gckOS_WriteMemory( + Hardware->os, + buffer + 9, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + )); + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the PIPESELECT command. */ + *Bytes = 40; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return status; +} + +gceSTATUS +gckHARDWARE_FlushMcfeMMU( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctUINT32 * Bytes + ) +{ + gceSTATUS status; + gctUINT32_PTR buffer; + gctUINT32 flushSize; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* MCFE with old mmu? nonsense!. */ + gcmkASSERT(Hardware->mmuVersion > 0); + + flushSize = 4 * 6; + + if (Logical) + { + if (*Bytes < flushSize) + { + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + buffer = (gctUINT32_PTR)Logical; + + /* Flush MMU cache. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0061) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = (((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) & ((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7)))); + + /* + * System channel can only take one command at a time, Trigger and + * SubmitJob are not required. + */ + + /* AQHiIdle to trigger. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0001) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = 0; + + /* SubmitJob. */ + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x16 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) (0x001 & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + } + + if (Bytes) + { + *Bytes = flushSize; + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_BuildVirtualAddress +** +** Build a virtual address. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** gctUINT32 Index +** Index into page table. +** +** gctUINT32 Offset +** Offset into page. +** +** OUTPUT: +** +** gctUINT32 * Address +** Pointer to a variable receiving te hardware address. +*/ +gceSTATUS +gckHARDWARE_BuildVirtualAddress( + IN gckHARDWARE Hardware, + IN gctUINT32 Index, + IN gctUINT32 Offset, + OUT gctUINT32 * Address + ) +{ + gcmkHEADER_ARG("Hardware=0x%x Index=%u Offset=%u", Hardware, Index, Offset); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(Address != gcvNULL); + + /* Build virtual address. */ + *Address = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:31) - (0 ? + 31:31) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:31) - (0 ? + 31:31) + 1))))))) << (0 ? + 31:31))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 31:31) - (0 ? + 31:31) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:31) - (0 ? 31:31) + 1))))))) << (0 ? 31:31))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 30:0) - (0 ? + 30:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 30:0) - (0 ? + 30:0) + 1))))))) << (0 ? + 30:0))) | (((gctUINT32) ((gctUINT32) (Offset | (Index << 12)) & ((gctUINT32) ((((1 ? + 30:0) - (0 ? + 30:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 30:0) - (0 ? 30:0) + 1))))))) << (0 ? 30:0))); + + /* Success. */ + gcmkFOOTER_ARG("*Address=0x%08x", *Address); + return gcvSTATUS_OK; +} + +gceSTATUS +gckHARDWARE_GetIdle( + IN gckHARDWARE Hardware, + IN gctBOOL Wait, + OUT gctUINT32 * Data + ) +{ + gceSTATUS status; + gctUINT32 idle = 0; + gctINT retry, poll, pollCount; + gctUINT32 address; + + gcmkHEADER_ARG("Hardware=0x%x Wait=%d", Hardware, Wait); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(Data != gcvNULL); + + + /* If we have to wait, try 100 polls per millisecond. */ + pollCount = Wait ? 100 : 1; + + /* At most, try for 1 second. */ + for (retry = 0; retry < 1000; ++retry) + { + /* If we have to wait, try 100 polls per millisecond. */ + for (poll = pollCount; poll > 0; --poll) + { + /* Read register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00004, &idle)); + + /* Read the current FE address. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00664, + &address)); + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00664, + &address)); + + /* See if we have to wait for FE idle. */ + if (_IsGPUIdle(idle) + && (address == Hardware->lastEnd + 8) + ) + { + /* FE is idle. */ + break; + } + } + + /* Check if we need to wait for FE and FE is busy. */ + if (Wait && !_IsGPUIdle(idle)) + { + /* Wait a little. */ + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "%s: Waiting for idle: 0x%08X", + __FUNCTION__, idle); + + gcmkVERIFY_OK(gckOS_Delay(Hardware->os, 1)); + } + else + { + break; + } + } + + /* Return idle to caller. */ + *Data = idle; + +#if defined(EMULATOR) + /* Wait a little while until CModel FE gets END. + * END is supposed to be appended by caller. + */ + gckOS_Delay(Hardware->os, 100); +#endif + + /* Success. */ + gcmkFOOTER_ARG("*Data=0x%08x", *Data); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/* Flush the caches. */ +gceSTATUS +gckHARDWARE_Flush( + IN gckHARDWARE Hardware, + IN gceKERNEL_FLUSH Flush, + IN gctPOINTER Logical, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT32 pipe; + gctUINT32 flush = 0; + gctUINT32 flushVST = 0; + gctBOOL flushTileStatus; + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + gctBOOL halti5; + gctBOOL flushICache; + gctBOOL flushTXDescCache; + gctBOOL flushTFB; + gctBOOL hwTFB; + gctBOOL blt; + gctBOOL peTSFlush; + gctBOOL multiCluster; + + gcmkHEADER_ARG("Hardware=0x%x Flush=0x%x Logical=0x%x *Bytes=%lu", + Hardware, Flush, Logical, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + /* Get current pipe. */ + pipe = Hardware->kernel->command->pipeSelect; + + halti5 = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_HALTI5); + + hwTFB = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_HW_TFB); + + blt = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_BLT_ENGINE); + multiCluster = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_MULTI_CLUSTER); + + peTSFlush = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_PE_TILE_CACHE_FLUSH_FIX); + + /* Flush tile status cache. */ + flushTileStatus = Flush & gcvFLUSH_TILE_STATUS; + + /* Flush Icache for halti5 hardware as we dont do it when program or context switches*/ + flushICache = (Flush & gcvFLUSH_ICACHE) && halti5; + + /* Flush texture descriptor cache */ + flushTXDescCache = Flush & gcvFLUSH_TXDESC; + + /* Flush USC cache for TFB client */ + flushTFB = (Flush & gcvFLUSH_TFBHEADER) && hwTFB; + + /* Flush TFB for vertex buffer */ + if (Flush & gcvFLUSH_VERTEX) + { + if (hwTFB) + { + flushTFB = gcvTRUE; + } + + if (multiCluster) + { + flush |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 14:14) - (0 ? + 14:14) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 14:14) - (0 ? + 14:14) + 1))))))) << (0 ? + 14:14))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 14:14) - (0 ? + 14:14) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 14:14) - (0 ? 14:14) + 1))))))) << (0 ? 14:14))); + } + } + + /* Flush 3D color cache. */ + if ((Flush & gcvFLUSH_COLOR) && (pipe == 0x0)) + { + flush |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))); + } + + /* Flush 3D depth cache. */ + if ((Flush & gcvFLUSH_DEPTH) && (pipe == 0x0)) + { + flush |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + + /* Flush 3D texture cache. */ + if ((Flush & gcvFLUSH_TEXTURE) && (pipe == 0x0)) + { + flush |= multiCluster ? + 0 : ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2))); + flushVST = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))); + } + + /* Flush 2D cache. */ + if ((Flush & gcvFLUSH_2D) && (pipe == 0x1)) + { + flush |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))); + } + + /* Flush L2 cache. */ + if ((Flush & gcvFLUSH_L2) && (pipe == 0x0)) + { + flush |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:6) - (0 ? + 6:6) + 1))))))) << (0 ? + 6:6))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:6) - (0 ? 6:6) + 1))))))) << (0 ? 6:6))); + } + + /* Vertex buffer and texture could be touched by SHL1 for SSBO and image load/store */ + if ((Flush & (gcvFLUSH_VERTEX | gcvFLUSH_TEXTURE)) && (pipe == 0x0)) + { + if (Hardware->identity.chipModel != 0x8000 || Hardware->identity.chipRevision != 0x7120 || !Hardware->options.enableNNTPParallel) + { + flush |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 10:10) - (0 ? + 10:10) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 10:10) - (0 ? + 10:10) + 1))))))) << (0 ? + 10:10))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 10:10) - (0 ? + 10:10) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 10:10) - (0 ? 10:10) + 1))))))) << (0 ? 10:10))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 11:11) - (0 ? + 11:11) + 1))))))) << (0 ? + 11:11))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 11:11) - (0 ? 11:11) + 1))))))) << (0 ? 11:11))); + } + else + { + flush |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 10:10) - (0 ? + 10:10) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 10:10) - (0 ? + 10:10) + 1))))))) << (0 ? + 10:10))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 10:10) - (0 ? + 10:10) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 10:10) - (0 ? 10:10) + 1))))))) << (0 ? 10:10))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 11:11) - (0 ? + 11:11) + 1))))))) << (0 ? + 11:11))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 11:11) - (0 ? 11:11) + 1))))))) << (0 ? 11:11))); + } + } + + /* See if there is a valid flush. */ + if ((flush == 0) && + (flushTileStatus == gcvFALSE) && + (flushICache == gcvFALSE) && + (flushTXDescCache == gcvFALSE) && + (flushTFB == gcvFALSE)) + { + if (Bytes != gcvNULL) + { + /* No bytes required. */ + *Bytes = 0; + } + } + else + { + gctBOOL appendNop = gcvFALSE; + gctUINT32 reserveBytes = 0; + gctBOOL txCacheFix = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_TEX_CACHE_FLUSH_FIX) + ? gcvTRUE : gcvFALSE; + + /* Determine reserve bytes. */ + if (!txCacheFix || flushICache || flushTXDescCache) + { + /* Semaphore/Stall */ + reserveBytes += blt ? (8 * gcmSIZEOF(gctUINT32)) : (4 * gcmSIZEOF(gctUINT32)); + } + + if (flush) + { + reserveBytes += 2 * gcmSIZEOF(gctUINT32); + } + + if (flushVST) + { + reserveBytes += 2 * gcmSIZEOF(gctUINT32); + } + + if (flushTileStatus) + { + reserveBytes += (!peTSFlush && blt) ? 6 * gcmSIZEOF(gctUINT32) : 2 * gcmSIZEOF(gctUINT32); + } + + if (flushICache) + { + reserveBytes += 2 * gcmSIZEOF(gctUINT32); + } + + if (flushTXDescCache) + { + reserveBytes += 2 * gcmSIZEOF(gctUINT32); + } + + if (flushTFB) + { + reserveBytes += 2 * gcmSIZEOF(gctUINT32); + } + + /* Semaphore/Stall */ + reserveBytes += blt ? (8 * gcmSIZEOF(gctUINT32)) : (4 * gcmSIZEOF(gctUINT32)); + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_MCFE) && (reserveBytes & 8)) + { + appendNop = gcvTRUE; + reserveBytes += 8; + } + + /* Copy to command queue. */ + if (Logical != gcvNULL) + { + if (*Bytes < reserveBytes) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + if (!txCacheFix || flushICache || flushTXDescCache) + { + if (blt) + { + /* Semaphore from FE to BLT. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to BLT. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + else + { + /* Semaphore. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + } + + if (flush) + { + /* Append LOAD_STATE to AQFlush. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ = flush; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, "0x%x: FLUSH 0x%x", logical - 1, flush); + } + + if (flushVST) + { + /* Append LOAD_STATE to AQFlush. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ = flushVST; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, "0x%x: FLUSH 0x%x", logical - 1, flush); + } + + if (flushTileStatus) + { + if (!peTSFlush && blt) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502B) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + else + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0594) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%x: FLUSH TILE STATUS 0x%x", logical - 1, logical[-1]); + } + + if (flushICache) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x022C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%x: FLUSH Icache 0x%x", logical - 1, logical[-1]); + + } + + if (flushTXDescCache) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x5311) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:28) - (0 ? + 31:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:28) - (0 ? + 31:28) + 1))))))) << (0 ? + 31:28))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 31:28) - (0 ? + 31:28) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:28) - (0 ? 31:28) + 1))))))) << (0 ? 31:28))); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%x: FLUSH Icache 0x%x", logical - 1, logical[-1]); + + } + + if (flushTFB) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x7003) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = 0x1; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%x: FLUSH TFB cache 0x%x", logical - 1, logical[-1]); + + } + + if (blt) + { + /* Semaphore from FE to BLT. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall from FE to BLT. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x10 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + else + { + /* Semaphore. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + + /* Stall. */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); + } + if (appendNop) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + } + } + + if (Bytes != gcvNULL) + { + /* bytes required. */ + *Bytes = reserveBytes; + } + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_SetFastClear( + IN gckHARDWARE Hardware, + IN gctINT Enable, + IN gctINT Compression + ) +{ +#if gcdENABLE_3D + gctUINT32 debug; + gceSTATUS status; + gceCOMPRESSION_OPTION compression = (Compression == -1) ? gcvCOMPRESSION_OPTION_DEFAULT : (gceCOMPRESSION_OPTION)Compression; + + gcmkHEADER_ARG("Hardware=0x%x Enable=%d Compression=%d", + Hardware, Enable, Compression); + + /* Only process if fast clear is available. */ + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_FAST_CLEAR)) + { + if (Enable == -1) + { + /* Determine automatic value for fast clear. */ + Enable = ((Hardware->identity.chipModel != gcv500) + || (Hardware->identity.chipRevision >= 3) + ) ? 1 : 0; + } + + if (compression == gcvCOMPRESSION_OPTION_DEFAULT) + { + /* Determine automatic value for compression. */ + if (Enable) + { + if (gcvSTATUS_FALSE == gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_ZCOMPRESSION)) + { + compression &= ~gcvCOMPRESSION_OPTION_DEPTH; + } + } + else + { + compression = gcvCOMPRESSION_OPTION_NONE; + } + } + + /* Read AQMemoryDebug register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00414, &debug)); + + /* Set fast clear bypass. */ + debug = ((((gctUINT32) (debug)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 20:20) - (0 ? + 20:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 20:20) - (0 ? + 20:20) + 1))))))) << (0 ? + 20:20))) | (((gctUINT32) ((gctUINT32) (Enable == 0) & ((gctUINT32) ((((1 ? + 20:20) - (0 ? + 20:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 20:20) - (0 ? 20:20) + 1))))))) << (0 ? 20:20))); + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_BUG_FIXES7) || + (Hardware->identity.chipModel >= gcv4000)) + { + /* Set compression bypass. */ + debug = ((((gctUINT32) (debug)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 21:21) - (0 ? + 21:21) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 21:21) - (0 ? + 21:21) + 1))))))) << (0 ? + 21:21))) | (((gctUINT32) ((gctUINT32) ((gcvCOMPRESSION_OPTION_NONE == compression) ? + 1 : 0) & ((gctUINT32) ((((1 ? + 21:21) - (0 ? + 21:21) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 21:21) - (0 ? 21:21) + 1))))))) << (0 ? 21:21))); + } + + /* Write back AQMemoryDebug register. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00414, + debug)); + + /* Store fast clear and comprersison flags. */ + Hardware->options.allowFastClear = Enable; + Hardware->options.allowCompression = compression; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "FastClear=%d Compression=%d", Enable, Compression); + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +#else + return gcvSTATUS_OK; +#endif +} + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) +static gctCONST_STRING +_PowerEnum(gceCHIPPOWERSTATE State) +{ + const gctCONST_STRING baseStates[] = + { + "ON", + "IDLE", + "SUSPEND", + "OFF", + "ON[auto]", + }; + + const gctCONST_STRING broadcastStates[] = + { + "", + "IDLE[broadcast]", + "SUSPEND[broadcast]", + "OFF[broadcast]", + }; + + const gctCONST_STRING timeoutStates[] = + { + "", + "IDLE[timeout]", + "SUSPEND[timeout]", + "OFF[timeout]", + }; + + gcmSTATIC_ASSERT(gcvPOWER_ON == 0 && gcvPOWER_IDLE == 1 && + gcvPOWER_SUSPEND == 2 && gcvPOWER_OFF == 3 && + gcvPOWER_ON_AUTO == 4, + "array subscript does not match"); + + if (State & gcvPOWER_FLAG_BROADCAST) + { + return broadcastStates[State & ~gcvPOWER_FLAG_BROADCAST]; + } + else if (State & gcvPOWER_FLAG_TIMEOUT) + { + return timeoutStates[State & ~gcvPOWER_FLAG_TIMEOUT]; + } + else if ((State >= gcvPOWER_ON) && (State <= gcvPOWER_ON_AUTO)) + { + return baseStates[State - gcvPOWER_ON]; + } + + return "unknown"; +} +#endif + +static gceSTATUS +_PmClockOn( + IN gckHARDWARE Hardware, + OUT gctBOOL * RequireInit + ) +{ + gceSTATUS status; + + /* Turn on the power. */ + gcmkONERROR(gckOS_SetGPUPower(Hardware->os, Hardware->core, + gcvTRUE, gcvTRUE)); + + Hardware->clockState = Hardware->powerState = gcvTRUE; + + /* Check if GPU is present and awake. */ + while (_IsGPUPresent(Hardware) == gcvSTATUS_GPU_NOT_RESPONDING) + { + /* Turn off the power and clock. */ + gcmkONERROR(gckOS_SetGPUPower(Hardware->os, Hardware->core, + gcvFALSE, gcvFALSE)); + + Hardware->clockState = Hardware->powerState = gcvFALSE; + + /* Wait a little. */ + gckOS_Delay(Hardware->os, 1); + + /* Turn on the power and clock. */ + gcmkONERROR(gckOS_SetGPUPower(Hardware->os, Hardware->core, + gcvTRUE, gcvTRUE)); + + Hardware->clockState = Hardware->powerState = gcvTRUE; + + if (RequireInit) + { + *RequireInit = gcvTRUE; + } + } + +OnError: + return status; +} + +static gceSTATUS +_PmClockOff( + IN gckHARDWARE Hardware, + IN gctBOOL PowerState + ) +{ + gceSTATUS status; + + gcmkONERROR(gckOS_SetGPUPower(Hardware->os, Hardware->core, + gcvFALSE, PowerState)); + + Hardware->clockState = gcvFALSE; + Hardware->powerState = PowerState; + +OnError: + return status; +} + +static gceSTATUS +_PmClockControl( + IN gckHARDWARE Hardware, + IN gceCHIPPOWERSTATE State + ) +{ + gceSTATUS status; + gctUINT32 clock; + + static const gctUINT clocks[4] = + { + /* gcvPOWER_ON */ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:2) - (0 ? + 8:2) + 1))))))) << (0 ? + 8:2))) | (((gctUINT32) ((gctUINT32) (64) & ((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 8:2) - (0 ? 8:2) + 1))))))) << (0 ? 8:2))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))), + + /* gcvPOWER_IDLE */ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:2) - (0 ? + 8:2) + 1))))))) << (0 ? + 8:2))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 8:2) - (0 ? 8:2) + 1))))))) << (0 ? 8:2))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))), + + /* gcvPOWER_SUSPEND */ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:2) - (0 ? + 8:2) + 1))))))) << (0 ? + 8:2))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 8:2) - (0 ? 8:2) + 1))))))) << (0 ? 8:2))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))), + + /* gcvPOWER_OFF */ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:2) - (0 ? + 8:2) + 1))))))) << (0 ? + 8:2))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 8:2) - (0 ? 8:2) + 1))))))) << (0 ? 8:2))) | + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))), + + }; + + clock = clocks[State]; + +#if gcdENABLE_FSCALE_VAL_ADJUST + if (State == gcvPOWER_ON) + { + clock = ((((gctUINT32) (clock)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:2) - (0 ? + 8:2) + 1))))))) << (0 ? + 8:2))) | (((gctUINT32) ((gctUINT32) (Hardware->powerOnFscaleVal) & ((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 8:2) - (0 ? 8:2) + 1))))))) << (0 ? 8:2))); + } +#endif + + if (Hardware->clockState && Hardware->powerState +#if gcdDVFS + /* Don't touch clock control if dynamic frequency scaling is available. */ + && !gckHARDWARE_IsFeatureAvailable(Hardware, + gcvFEATURE_DYNAMIC_FREQUENCY_SCALING) +#endif + ) + { + if ((State == gcvPOWER_OFF) && + (Hardware->identity.chipModel == gcv4000) && + ((Hardware->identity.chipRevision == 0x5208) || + (Hardware->identity.chipRevision == 0x5222))) + { + clock &= ~2U; + } + + /* Write the clock control register. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, + 0x00000, + clock)); + + clock = ((((gctUINT32) (clock)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))); + + /* Done loading the frequency scaler. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, + 0x00000, + clock)); + } + + return gcvSTATUS_OK; + +OnError: + return status; +} + +static gceSTATUS +_PmInitializeGPU( + IN gckHARDWARE Hardware, + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + + gctBOOL hwMmuDisabled = gcvTRUE; + + /* VIV for 8MM_EVK, maybe power off is failed, the GPU still has been power on, + so we need to check the mmu enable flag to see if we need to dummy draw */ + if (_IsHardwareMatch(Hardware, gcv600, 0x4653)) + { + if (Hardware->options.secureMode == gcvSECURE_IN_NORMAL) + { + gctUINT32 regMmuCtrl = 0; + gcmkONERROR(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x00388, + ®MmuCtrl + )); + + hwMmuDisabled = ((((((gctUINT32) (regMmuCtrl)) >> (0 ? 0:0)) & ((gctUINT32) ((((1 ? 0:0) - (0 ? 0:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1)))))) ) == 0x1) + ? gcvFALSE + : gcvTRUE; + } + else + { + gctUINT32 regMmuCtrl = 0; + + gcmkONERROR(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x0018C, + ®MmuCtrl + )); + + hwMmuDisabled = ((((((gctUINT32) (regMmuCtrl)) >> (0 ? 0:0)) & ((gctUINT32) ((((1 ? 0:0) - (0 ? 0:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1)))))) ) == 0x1) + ? gcvFALSE + : gcvTRUE; + } + } + + if(hwMmuDisabled) + { + Command->dummyDraw = gcvTRUE; + } + + /* Initialize hardware. */ + gcmkONERROR(gckHARDWARE_InitializeHardware(Hardware)); + + gcmkONERROR(gckHARDWARE_SetFastClear(Hardware, + Hardware->options.allowFastClear, + Hardware->options.allowCompression)); + + /* Force the command queue to reload the next context. */ + Command->currContext = gcvNULL; + +OnError: + return status; +} + +/* + * Notice: this function may return gcvSTATUS_NOT_READY, which is not an error, + * but either not success. + */ +static gceSTATUS +_PmStallCommand( + gckHARDWARE Hardware, + gckCOMMAND Command, + gctBOOL Broadcast + ) +{ + gceSTATUS status; + gctBOOL idle; + + if (Broadcast) + { + /* Check for idle. */ + gcmkONERROR(gckHARDWARE_QueryIdle(Hardware, &idle)); + + if (!idle) + { + status = gcvSTATUS_CHIP_NOT_READY; + goto OnError; + } + } + else + { + /* Wait to finish all commands. */ + status = gckCOMMAND_Stall(Command, gcvTRUE); + + if (!gcmIS_SUCCESS(status)) + { + goto OnError; + } + + for (;;) + { + gcmkONERROR(gckHARDWARE_QueryIdle(Hardware, &idle)); + + if (idle) + { + break; + } + + gcmkVERIFY_OK(gckOS_Delay(Hardware->os, 1)); + } + } + +OnError: + return status; +} + +static gceSTATUS +_PmFlushCache( + gckHARDWARE Hardware, + gckCOMMAND Command + ) +{ + gceSTATUS status = gcvSTATUS_OK; + + if (Hardware->clockState == gcvFALSE) + { + /* Turn on the GPU power. */ + gcmkONERROR(gckOS_SetGPUPower(Hardware->os, Hardware->core, + gcvTRUE, gcvTRUE)); + + Hardware->clockState = gcvTRUE; + + /* Clock control, to ON state. */ + gcmkONERROR(_PmClockControl(Hardware, gcvPOWER_ON)); + } + + if (_IsHardwareMatch(Hardware, gcv400, 0x4645)) + { + gcmkONERROR(gckCOMMAND_Start(Command)); + + gcmkONERROR(_FlushCache(Hardware, Command)); + gckOS_Delay(gcvNULL, 1); + + /* Stop the command parser. */ + gcmkONERROR(gckCOMMAND_Stop(Command)); + } + else + { + gcmkONERROR(gckFUNCTION_Execute(&Hardware->functions[gcvFUNCTION_EXECUTION_FLUSH])); + gckOS_Delay(gcvNULL, 1); + } + +OnError: + return status; +} + +/* + * Put power to on direction: + * Off -> Suspend + * Off -> Idle + * Off -> On + * Suspend -> Idle + * Suspend -> On + * Idle -> On + */ +static gceSTATUS +_PmSetPowerOnDirection( + IN gckHARDWARE Hardware, + IN gceCHIPPOWERSTATE State + ) +{ + gceSTATUS status; + gckCOMMAND command = Hardware->kernel->command; + gctBOOL clockOn = gcvFALSE; + gctBOOL requireInit = gcvFALSE; + + switch (Hardware->chipPowerState) + { + case gcvPOWER_OFF: + if (State == gcvPOWER_SUSPEND) + { + gcmkONERROR(_PmClockOn(Hardware, gcvNULL)); + clockOn = gcvTRUE; + + /* Clock control, put to suspend. */ + gcmkONERROR(_PmClockControl(Hardware, gcvPOWER_SUSPEND)); + + /* Initialize GPU. */ + gcmkONERROR(_PmInitializeGPU(Hardware, command)); + + /* Suspend: clock off, power on. */ + gcmkONERROR(_PmClockOff(Hardware, gcvTRUE)); + break; + } + + requireInit = gcvTRUE; + /* FALLTHRU */ + + case gcvPOWER_SUSPEND: + /* Clock on. */ + gcmkONERROR(_PmClockOn(Hardware, &requireInit)); + clockOn = gcvTRUE; + + /* Clock control, put to target state (On or idle). */ + gcmkONERROR(_PmClockControl(Hardware, State)); + + /* Delay. */ + gcmkONERROR(gckOS_Delay(Hardware->os, gcdPOWER_CONTROL_DELAY)); + + if (requireInit) + { + /* Initialize. */ + gcmkONERROR(_PmInitializeGPU(Hardware, command)); + } + + /* Start. */ + gcmkONERROR(gckCOMMAND_Start(command)); + break; + + case gcvPOWER_IDLE: + /* Clock control, put to ON state. */ + gcmkONERROR(_PmClockControl(Hardware, gcvPOWER_ON)); + break; + + default: + break; + } + + return gcvSTATUS_OK; + +OnError: + if (clockOn) + { + gctBOOL powerState = + (Hardware->chipPowerState == gcvPOWER_SUSPEND); + + _PmClockOff(Hardware, powerState); + } + + return status; +} + +/* + * Put power to off direction: + * On -> Idle + * On -> Suspend + * On -> Off + * Idle -> Suspend + * Idle -> Off + * Suspend -> Off + */ +static gceSTATUS +_PmSetPowerOffDirection( + IN gckHARDWARE Hardware, + IN gceCHIPPOWERSTATE State, + IN gctBOOL Broadcast + ) +{ + gceSTATUS status; + gckCOMMAND command = Hardware->kernel->command; + + switch (Hardware->chipPowerState) + { + case gcvPOWER_ON: + if(Hardware->kernel->threadInitialized == gcvTRUE) + { + /* Stall. */ + status = _PmStallCommand(Hardware, command, Broadcast); + + if (!gcmIS_SUCCESS(status)) + { + /* abort for error and NOT READY. */ + goto OnError; + } + } + + if (State == gcvPOWER_IDLE) + { + gcmkONERROR(_PmClockControl(Hardware, gcvPOWER_IDLE)); + break; + } + /* FALLTHRU */ + + case gcvPOWER_IDLE: + /* Stop. */ + gcmkONERROR(gckCOMMAND_Stop(command)); + + if (State == gcvPOWER_SUSPEND) + { + /* Clock control, put to suspend state. */ + gcmkONERROR(_PmClockControl(Hardware, gcvPOWER_SUSPEND)); + + /* Power on, clock off. */ + gcmkONERROR(_PmClockOff(Hardware, gcvTRUE)); + break; + } + /* FALLTHRU */ + + case gcvPOWER_SUSPEND: + if(Hardware->kernel->threadInitialized == gcvTRUE) + { + /* Flush. */ + gcmkONERROR(_PmFlushCache(Hardware, command)); + } + + /* Clock control. */ + gcmkONERROR(_PmClockControl(Hardware, gcvPOWER_OFF)); + + /* Power off, clock off. */ + gcmkONERROR(_PmClockOff(Hardware, gcvFALSE)); + + break; + + default: + break; + } + + return gcvSTATUS_OK; + +OnError: + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_SetPowerState +** +** Set GPU to a specified power state. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** gceCHIPPOWERSTATE State +** Power State. +** +*/ +gceSTATUS +gckHARDWARE_SetPowerState( + IN gckHARDWARE Hardware, + IN gceCHIPPOWERSTATE State + ) +{ + gceSTATUS status; + gckCOMMAND command = gcvNULL; + gckOS os; + gctBOOL acquired = gcvFALSE; + gctBOOL mutexAcquired = gcvFALSE; + gctBOOL broadcast = gcvFALSE; + gctBOOL timeout = gcvFALSE; + gceCHIPPOWERSTATE state = gcvPOWER_INVALID; + + /* + * Acquire globalSempahore when set to to global OFF, IDLE, SUSPEND, then + * can not switch to non-global state unless global ON comes and release + * the globalSemaphore. + */ + gctBOOL global = gcvFALSE; + gctBOOL globalAcquired = gcvFALSE; + + gcmkHEADER_ARG("Hardware=0x%x State=%d", Hardware, State); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "Switching to power state %d(%s)", + State, _PowerEnum(State)); +#endif + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + os = Hardware->os; + command = Hardware->kernel->command; + + /* Convert power state. */ + switch (State) + { + case gcvPOWER_ON: + case gcvPOWER_OFF: + case gcvPOWER_IDLE: + case gcvPOWER_SUSPEND: + global = gcvTRUE; + state = State; + break; + + case gcvPOWER_ON_AUTO: + state = gcvPOWER_ON; + break; + + case gcvPOWER_OFF_TIMEOUT: + case gcvPOWER_IDLE_TIMEOUT: + case gcvPOWER_SUSPEND_TIMEOUT: + timeout = gcvTRUE; + /* FALLTHRU */ + case gcvPOWER_OFF_BROADCAST: + case gcvPOWER_IDLE_BROADCAST: + case gcvPOWER_SUSPEND_BROADCAST: + broadcast = gcvTRUE; + state = State & ~(gcvPOWER_FLAG_BROADCAST | gcvPOWER_FLAG_TIMEOUT); + break; + + case gcvPOWER_INVALID: + /* Canceled, nothing to do. */ + status = gcvSTATUS_OK; + goto OnError; + + default: + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + if (Hardware->options.powerManagement == gcvFALSE && + (Hardware->chipPowerState == gcvPOWER_ON || state != gcvPOWER_ON)) + { + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + if (broadcast) + { + /* Try to acquire the power mutex. */ + status = gckOS_AcquireMutex(os, Hardware->powerMutex, 0); + + if (status == gcvSTATUS_TIMEOUT) + { + /* Pm in progress, abort. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + } + else + { + /* Acquire the power mutex. */ + status = gckOS_AcquireMutex(os, Hardware->powerMutex, gcvINFINITE); + } + + gcmkONERROR(status); + mutexAcquired = gcvTRUE; + + if (Hardware->chipPowerState == state) + { + /* No state change. */ + gckOS_ReleaseMutex(os, Hardware->powerMutex) + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + if (broadcast && + state == gcvPOWER_SUSPEND && Hardware->chipPowerState == gcvPOWER_OFF) + { + /* Do nothing, donot change chipPowerState. */ + gckOS_ReleaseMutex(os, Hardware->powerMutex) + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + if (timeout) + { + if (Hardware->nextPowerState == gcvPOWER_INVALID) + { + /* delayed power state change is canceled. */ + gckOS_ReleaseMutex(os, Hardware->powerMutex) + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + } + + if (global) + { + if (state != gcvPOWER_ON) + { + /* + * Switch to global non-ON (OFF, IDLE or SUSPEND), try acquire the + * global semaphore if it has not been acquired. + */ + status = gckOS_TryAcquireSemaphore(os, Hardware->globalSemaphore); + + if (status == gcvSTATUS_OK) + { + globalAcquired = gcvTRUE; + } + else if (status == gcvSTATUS_TIMEOUT) + { + /* + * Ignore and leave globalSemaphore not acquired in this try. + * In this condition, power state is changing between global + * OFF, SUSPEND and IDLE, which is allowed. + */ + gcmkASSERT(Hardware->chipPowerState != gcvPOWER_ON); + } + else + { + /* Other errors. */ + gcmkONERROR(status); + } + } + } + else + { + /* Try to acquire the global semaphore. */ + status = gckOS_TryAcquireSemaphore(os, Hardware->globalSemaphore); + + if (status == gcvSTATUS_TIMEOUT) + { + /* In global SUSPEND, IDLE, or OFF state. */ + gcmkONERROR(gckOS_ReleaseMutex(os, Hardware->powerMutex)); + mutexAcquired = gcvFALSE; + + if (broadcast) + { + /* Abort power state change. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + /* + * Only ON_AUTO can run here. + * ON_AUTO state can not be skipped, either can not change a global + * state. + * So we need to wait until global ON state here. + */ + gcmkONERROR(gckOS_AcquireSemaphore(os, Hardware->globalSemaphore)); + globalAcquired = gcvTRUE; + + /* Acquire the power mutex. */ + gcmkONERROR(gckOS_AcquireMutex(os, + Hardware->powerMutex, + gcvINFINITE)); + mutexAcquired = gcvTRUE; + + if (Hardware->chipPowerState == state) + { + /* Done. */ + status = gcvSTATUS_OK; + goto OnError; + } + } + else + { + /* Check error. */ + gcmkONERROR(status); + } + + /* We just check if in non-global state, but need not to acquire it. */ + gcmkONERROR(gckOS_ReleaseSemaphore(os, Hardware->globalSemaphore)); + globalAcquired = gcvFALSE; + } + + if (Hardware->chipPowerState == gcvPOWER_ON) + { + /* Switch from power ON to other non-runnable states. */ + if (broadcast) + { + gctINT32 atomValue; + + /* Try to acquire the semaphore to block/sync with commit. */ + status = gckOS_TryAcquireSemaphore(os, command->powerSemaphore); + + if (gcmIS_ERROR(status)) + { + status = gcvSTATUS_CHIP_NOT_READY; + goto OnError; + } + + acquired = gcvTRUE; + + /* Check commit atom, abort when commit is in progress. */ + gcmkONERROR(gckOS_AtomGet(Hardware->os, + command->atomCommit, + &atomValue)); + + if (atomValue > 0) + { + status = gcvSTATUS_CHIP_NOT_READY; + goto OnError; + } + } + else + { + /* Acquire/wait the semaphore to block/sync with command commit. */ + gcmkONERROR(gckOS_AcquireSemaphore(os, command->powerSemaphore)); + acquired = gcvTRUE; + } + } + + /* Do hardware power state change. */ + if (Hardware->chipPowerState < state) + { + /* On to off direction. */ + gcmkONERROR(_PmSetPowerOffDirection(Hardware, state, broadcast)); + } + else + { + /* Off to on direction. */ + gcmkONERROR(_PmSetPowerOnDirection(Hardware, state)); + } + + if (status == gcvSTATUS_CHIP_NOT_READY) + { + /* CHIP_NOT_READY is not an error, either not success. */ + goto OnError; + } + + if (state == gcvPOWER_ON) + { + /* Switched to power ON from other non-runnable states. */ + gcmkONERROR(gckOS_ReleaseSemaphore(os, command->powerSemaphore)); + acquired = gcvFALSE; + + if (global) + { + /* + * Global semaphore should be acquired already before, when + * global OFF, IDLE or SUSPEND. + */ + status = gckOS_TryAcquireSemaphore(os, Hardware->globalSemaphore); + if (status != gcvSTATUS_TIMEOUT && Hardware->isLastPowerGlobal) + { + gcmkPRINT("%s: global state error", __FUNCTION__); + } + + /* Switched to global ON, now release the global semaphore. */ + gcmkONERROR(gckOS_ReleaseSemaphore(os, Hardware->globalSemaphore)); + globalAcquired = gcvFALSE; + } + } + + gckSTATETIMER_Accumulate(&Hardware->powerStateCounter, + Hardware->chipPowerState); + + /* Save the new power state. */ + Hardware->chipPowerState = state; + Hardware->isLastPowerGlobal = global; + +#if gcdDVFS + if (state == gcvPOWER_ON && Hardware->kernel->dvfs) + { + gckDVFS_Start(Hardware->kernel->dvfs); + } +#endif + + if (!broadcast) + { + /* + * Cancel delayed power state change. + * Stop timer is not as good as set as no state change. Timer may run + * into this function already when try to to stop the timer. + */ + Hardware->nextPowerState = gcvPOWER_INVALID; + } + +#if gcdPOWEROFF_TIMEOUT + if (state == gcvPOWER_IDLE || state == gcvPOWER_SUSPEND) + { + /* Delayed power off. */ + Hardware->nextPowerState = gcvPOWER_OFF_TIMEOUT; + + /* Start a timer to power off GPU when GPU enters IDLE or SUSPEND. */ + gcmkVERIFY_OK(gckOS_StartTimer(os, Hardware->powerStateTimer, gcdPOWEROFF_TIMEOUT)); + } +#endif + + /* Release the power mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(os, Hardware->powerMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release semaphore. */ + gcmkVERIFY_OK(gckOS_ReleaseSemaphore(Hardware->os, + command->powerSemaphore)); + } + + if (globalAcquired) + { + gcmkVERIFY_OK(gckOS_ReleaseSemaphore(Hardware->os, + Hardware->globalSemaphore)); + } + + if (mutexAcquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Hardware->os, Hardware->powerMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_QueryPowerState +** +** Get GPU power state. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** gceCHIPPOWERSTATE* State +** Power State. +** +*/ +gceSTATUS +gckHARDWARE_QueryPowerState( + IN gckHARDWARE Hardware, + OUT gceCHIPPOWERSTATE* State + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(State != gcvNULL); + + /* Return the statue. */ + *State = Hardware->chipPowerState; + + /* Success. */ + gcmkFOOTER_ARG("*State=%d", *State); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_EnablePowerManagement +** +** Configure GPU power management function. +** Only used in driver initialization stage. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** gctBOOL Enable +** Power Mangement Enabling State. +** +*/ +gceSTATUS +gckHARDWARE_EnablePowerManagement( + IN gckHARDWARE Hardware, + IN gctBOOL Enable + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + gcmkVERIFY_OK( + gckOS_AcquireMutex(Hardware->os, Hardware->powerMutex, gcvINFINITE)); + + Hardware->options.powerManagement = Enable; + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Hardware->os, Hardware->powerMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_SetGpuProfiler +** +** Configure GPU profiler function. +** Only used in driver initialization stage. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** gctBOOL GpuProfiler +** GOU Profiler State. +** +*/ +gceSTATUS +gckHARDWARE_SetGpuProfiler( + IN gckHARDWARE Hardware, + IN gctBOOL GpuProfiler + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + if (GpuProfiler == gcvTRUE) + { + gctUINT32 data = 0; + + /* Need to disable clock gating when doing profiling. */ + gcmkVERIFY_OK( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00100, + &data)); + + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + + gcmkVERIFY_OK( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00100, + data)); + } + + Hardware->options.gpuProfiler= GpuProfiler; + + if (GpuProfiler == gcvTRUE) + { + Hardware->waitCount = 200 * 100; + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +#if gcdENABLE_FSCALE_VAL_ADJUST +gceSTATUS +gckHARDWARE_SetFscaleValue( + IN gckHARDWARE Hardware, + IN gctUINT32 FscaleValue, + IN gctUINT32 ShaderFscaleValue + ) +{ + gceSTATUS status; + gctUINT32 clock; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Hardware=0x%x FscaleValue=%d", Hardware, FscaleValue); + + gcmkVERIFY_ARGUMENT(FscaleValue > 0 && FscaleValue <= 64); + + gcmkONERROR( + gckOS_AcquireMutex(Hardware->os, Hardware->powerMutex, gcvINFINITE)); + acquired = gcvTRUE; + + Hardware->powerOnFscaleVal = FscaleValue; + + if (Hardware->chipPowerState == gcvPOWER_ON) + { + gctUINT32 data; + + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + &data)); + + /* Disable all clock gating. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:6) - (0 ? + 6:6) + 1))))))) << (0 ? + 6:6))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:6) - (0 ? 6:6) + 1))))))) << (0 ? 6:6))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:8) - (0 ? + 8:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:8) - (0 ? + 8:8) + 1))))))) << (0 ? + 8:8))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 8:8) - (0 ? + 8:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 8:8) - (0 ? 8:8) + 1))))))) << (0 ? 8:8))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 11:11) - (0 ? + 11:11) + 1))))))) << (0 ? + 11:11))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 11:11) - (0 ? 11:11) + 1))))))) << (0 ? 11:11))))); + + /* Scale the core clock. */ + clock = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:2) - (0 ? + 8:2) + 1))))))) << (0 ? + 8:2))) | (((gctUINT32) ((gctUINT32) (FscaleValue) & ((gctUINT32) ((((1 ? + 8:2) - (0 ? + 8:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 8:2) - (0 ? 8:2) + 1))))))) << (0 ? 8:2))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + clock)); + + /* Done loading the frequency scaler. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + ((((gctUINT32) (clock)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))))); + + /* A option to support shader clock scaling. */ + if (ShaderFscaleValue != ~0U && ShaderFscaleValue > 0 && ShaderFscaleValue < 64) + { + /* Scale the shader clock. */ + clock = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:17) - (0 ? + 17:17) + 1))))))) << (0 ? + 17:17))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:17) - (0 ? 17:17) + 1))))))) << (0 ? 17:17))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:1) - (0 ? + 7:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:1) - (0 ? + 7:1) + 1))))))) << (0 ? + 7:1))) | (((gctUINT32) ((gctUINT32) (ShaderFscaleValue) & ((gctUINT32) ((((1 ? + 7:1) - (0 ? + 7:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:1) - (0 ? 7:1) + 1))))))) << (0 ? 7:1))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + clock)); + + /* Done loading the frequency scaler. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + ((((gctUINT32) (clock)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))))); + } + + /* Restore all clock gating. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + Hardware->powerBaseAddress + + 0x00104, + data)); + } + + gcmkVERIFY(gckOS_ReleaseMutex(Hardware->os, Hardware->powerMutex)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gcmkVERIFY(gckOS_ReleaseMutex(Hardware->os, Hardware->powerMutex)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_GetFscaleValue( + IN gckHARDWARE Hardware, + IN gctUINT * FscaleValue, + IN gctUINT * MinFscaleValue, + IN gctUINT * MaxFscaleValue + ) +{ + *FscaleValue = Hardware->powerOnFscaleVal; + *MinFscaleValue = Hardware->minFscaleValue; + *MaxFscaleValue = 64; + + return gcvSTATUS_OK; +} + +gceSTATUS +gckHARDWARE_SetMinFscaleValue( + IN gckHARDWARE Hardware, + IN gctUINT MinFscaleValue + ) +{ + if (MinFscaleValue >= 1 && MinFscaleValue <= 64) + { + Hardware->minFscaleValue = MinFscaleValue; + } + + return gcvSTATUS_OK; +} +#endif + +gceSTATUS +gckHARDWARE_QueryIdle( + IN gckHARDWARE Hardware, + OUT gctBOOL_PTR IsIdle + ) +{ + gceSTATUS status; + gctUINT32 idle; +#if !gcdSECURITY + gctUINT32 address; +#endif + gctBOOL isIdle = gcvFALSE; + +#if gcdINTERRUPT_STATISTIC + gckEVENT eventObj = Hardware->kernel->eventObj; + gctINT32 pendingInterrupt; +#endif + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + +#if gcdCAPTURE_ONLY_MODE + *IsIdle = gcvTRUE; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +#endif + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(IsIdle != gcvNULL); + + do + { + /* We are idle when the power is not ON. */ + if (Hardware->chipPowerState != gcvPOWER_ON) + { + isIdle = gcvTRUE; + break; + } + + if (Hardware->mcFE) + { + gctBOOL isIdle; + + gcmkONERROR(gckMCFE_HardwareIdle(Hardware, &isIdle)); + + if(!isIdle) + { + break; + } + } + else + { + /* Read idle register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00004, &idle)); + + /* Pipe must be idle. */ + if ((idle | (1 << 14)) != 0x7ffffffe) + { + /* Something is busy. */ + break; + } + +#if gcdSECURITY + isIdle = gcvTRUE; + break; +#else + /* Read the current FE address. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00664, + &address)); + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00664, + &address)); + + /* Test if address is inside the last WAIT/LINK sequence. */ + if ((address < Hardware->lastWaitLink) || + (address >= (gctUINT64)Hardware->lastWaitLink + 16)) + { + /* FE is not in WAIT/LINK yet. */ + break; + } +#endif + } /* end of else */ + +#if gcdINTERRUPT_STATISTIC + gcmkONERROR(gckOS_AtomGet( + Hardware->os, + eventObj->interruptCount, + &pendingInterrupt + )); + + if (pendingInterrupt) + { + /* Pending interrupts, not idle. */ + break; + } + + if (Hardware->asyncFE) + { + gckEVENT asyncEvent = Hardware->kernel->asyncEvent; + + gcmkONERROR(gckOS_AtomGet( + Hardware->os, + asyncEvent->interruptCount, + &pendingInterrupt + )); + + if (pendingInterrupt) + { + /* Pending async FE interrupts, not idle. */ + break; + } + } +#endif + + /* Is really idle. */ + isIdle = gcvTRUE; + } + while (gcvFALSE); + + *IsIdle = isIdle; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** Handy macros that will help in reading those debug registers. +*/ +#define gcmkREAD_DEBUG_REGISTER_PART1(control, block, index, data) \ + gcmkONERROR(\ + gckOS_WriteRegisterEx(Hardware->os, \ + Hardware->core, \ + GC_DEBUG_CONTROL##control##_Address, \ + gcmSETFIELD(0, \ + GC_DEBUG_CONTROL##control, \ + block, \ + index))); \ + gcmkONERROR(\ + gckOS_ReadRegisterEx(Hardware->os, \ + Hardware->core, \ + GC_DEBUG_SIGNALS_##block##_Address, \ + &profiler_part1->data)) + +#define gcmkREAD_DEBUG_REGISTER_PART2(control, block, index, data) \ + gcmkONERROR(\ + gckOS_WriteRegisterEx(Hardware->os, \ + Hardware->core, \ + GC_DEBUG_CONTROL##control##_Address, \ + gcmSETFIELD(0, \ + GC_DEBUG_CONTROL##control, \ + block, \ + index))); \ + gcmkONERROR(\ + gckOS_ReadRegisterEx(Hardware->os, \ + Hardware->core, \ + GC_DEBUG_SIGNALS_##block##_Address, \ + &profiler_part2->data)) + +#define gcmkREAD_DEBUG_REGISTER_N(control, block, index, data) \ + gcmkONERROR(\ + gckOS_WriteRegisterEx(Hardware->os, \ + Hardware->core, \ + GC_DEBUG_CONTROL##control##_Address, \ + gcmSETFIELD(0, \ + GC_DEBUG_CONTROL##control, \ + block, \ + index))); \ + gcmkONERROR(\ + gckOS_ReadRegisterEx(Hardware->os, \ + Hardware->core, \ + GC_DEBUG_SIGNALS_##block##_Address, \ + &data)) + +#define gcmkRESET_DEBUG_REGISTER(control, block, value) \ + gcmkONERROR(\ + gckOS_WriteRegisterEx(Hardware->os, \ + Hardware->core, \ + GC_DEBUG_CONTROL##control##_Address, \ + gcmSETFIELD(0, \ + GC_DEBUG_CONTROL##control, \ + block, \ + value))); \ + gcmkONERROR(\ + gckOS_WriteRegisterEx(Hardware->os, \ + Hardware->core, \ + GC_DEBUG_CONTROL##control##_Address, \ + gcmSETFIELD(0, \ + GC_DEBUG_CONTROL##control, \ + block, \ + 0))) + +static gctUINT32 +CalcDelta( + IN gctUINT32 newval, + IN gctUINT32 oldval + ) +{ + if (newval >= oldval) + { + return newval - oldval; + } + else + { + return (gctUINT32)((gctUINT64)newval + 0x100000000ll - oldval); + } +} + + +#if USE_SW_RESET +#define gcmkRESET_PROFILE_DATA_PART1(counterName) \ + temp = profiler_part1->counterName; \ + profiler_part1->counterName = CalcDelta(temp, Context->preProfiler_part1.counterName); \ + Context->preProfiler_part1.counterName = temp +#endif + +#define gcmkUPDATE_PROFILE_DATA_PART1(data) \ + profilerHistroy_part1->data += profiler_part1->data +#define gcmkUPDATE_PROFILE_DATA_PART2(data) \ + profilerHistroy_part2->data += profiler_part2->data + +gceSTATUS +gckHARDWARE_QueryContextProfile( + IN gckHARDWARE Hardware, + IN gctBOOL Reset, + IN gckCONTEXT Context, + OUT gcsPROFILER_COUNTERS_PART1 * Counters_part1, + OUT gcsPROFILER_COUNTERS_PART2 * Counters_part2 +) +{ + gceSTATUS status; + gckCOMMAND command = Hardware->kernel->command; + gcsPROFILER_COUNTERS_PART1 * profiler_part1 = Counters_part1; + gcsPROFILER_COUNTERS_PART2 * profiler_part2 = Counters_part2; + + gcmkHEADER_ARG("Hardware=0x%x Counters_part1=0x%x, Counters_part2=0x%x", Hardware, Counters_part1, Counters_part2); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + if (!Context) + { + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + /* Acquire the context sequnence mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + command->os, command->mutexContextSeq, gcvINFINITE + )); + + /* Read the counters. */ + if (Counters_part1) + { + gcmkVERIFY_OK(gckOS_MemCopy( + profiler_part1, &Context->histroyProfiler_part1, gcmSIZEOF(gcsPROFILER_COUNTERS_PART1) + )); + } + else if (Counters_part2) + { + gcmkVERIFY_OK(gckOS_MemCopy( + profiler_part2, &Context->histroyProfiler_part2, gcmSIZEOF(gcsPROFILER_COUNTERS_PART2) + )); + } + + /* Reset counters. */ + if (Reset) + { + if (Counters_part1) + { + gcmkVERIFY_OK(gckOS_ZeroMemory( + &Context->histroyProfiler_part1, gcmSIZEOF(gcsPROFILER_COUNTERS_PART1) + )); + } + else if (Counters_part2) + { + gcmkVERIFY_OK(gckOS_ZeroMemory( + &Context->histroyProfiler_part2, gcmSIZEOF(gcsPROFILER_COUNTERS_PART2) + )); + } + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex( + command->os, command->mutexContextSeq + )); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_UpdateContextProfile( + IN gckHARDWARE Hardware, + IN gckCONTEXT Context +) +{ + gceSTATUS status; + gcsPROFILER_COUNTERS_PART1 * profiler_part1 = &Context->latestProfiler_part1; + gcsPROFILER_COUNTERS_PART1 * profilerHistroy_part1 = &Context->histroyProfiler_part1; + gcsPROFILER_COUNTERS_PART2 * profiler_part2 = &Context->latestProfiler_part2; + gcsPROFILER_COUNTERS_PART2 * profilerHistroy_part2 = &Context->histroyProfiler_part2; + gceCHIPMODEL chipModel; + gctUINT32 chipRevision; + gctUINT32 i; + gctUINT32 resetValue = 0xF; + gctBOOL newCounters0 = gcvFALSE; + gctUINT32 clock; + gctUINT32 colorKilled = 0, colorDrawn = 0, depthKilled = 0, depthDrawn = 0; + gctUINT32 totalRead, totalWrite; + gctUINT32 mc_axi_max_min_latency; + gctUINT32 temp; + gckCOMMAND command = Hardware->kernel->command; + gctBOOL mutexAcquired = gcvFALSE; + + gcmkHEADER_ARG("Hardware=0x%x Context=0x%x", Hardware, Context); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_OBJECT(Context, gcvOBJ_CONTEXT); + + /* Acquire the context sequnence mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + command->os, command->mutexContextSeq, gcvINFINITE + )); + mutexAcquired = gcvTRUE; + + chipModel = Hardware->identity.chipModel; + chipRevision = Hardware->identity.chipRevision; + if ((chipModel == gcv5000 && chipRevision == 0x5434) || (chipModel == gcv3000 && chipRevision == 0x5435)) + { + resetValue = 0xFF; + newCounters0 = gcvTRUE; + } + if (chipModel == gcv2100 || chipModel == gcv2000 || chipModel == gcv880) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00078, + &profiler_part2->hi_total_idle_cycle_count)); + + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00438, + &profiler_part2->hi_total_cycle_count)); + } + else + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x0007C, + &profiler_part2->hi_total_idle_cycle_count)); + + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00078, + &profiler_part2->hi_total_cycle_count)); + } + gcmkUPDATE_PROFILE_DATA_PART2(hi_total_cycle_count); + gcmkUPDATE_PROFILE_DATA_PART2(hi_total_idle_cycle_count); + + /* Read clock control register. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + &clock)); + + profiler_part2->hi_total_read_8B_count = 0; + profiler_part2->hi_total_write_8B_count = 0; + profiler_part2->hi_total_readOCB_16B_count = 0; + profiler_part2->hi_total_writeOCB_16B_count = 0; + profiler_part1->pe0_pixel_count_drawn_by_color_pipe = 0; + profiler_part1->pe0_pixel_count_drawn_by_depth_pipe = 0; + profiler_part1->pe0_pixel_count_killed_by_color_pipe = 0; + profiler_part1->pe0_pixel_count_killed_by_depth_pipe = 0; + profiler_part1->pe1_pixel_count_drawn_by_color_pipe = 0; + profiler_part1->pe1_pixel_count_drawn_by_depth_pipe = 0; + profiler_part1->pe1_pixel_count_killed_by_color_pipe = 0; + profiler_part1->pe1_pixel_count_killed_by_depth_pipe = 0; + + /* Walk through all avaiable pixel pipes. */ + for (i = 0; i < Hardware->identity.pixelPipes; ++i) + { + /* Select proper pipe. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + ((((gctUINT32) (clock)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:20) - (0 ? + 23:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:20) - (0 ? + 23:20) + 1))))))) << (0 ? + 23:20))) | (((gctUINT32) ((gctUINT32) (i) & ((gctUINT32) ((((1 ? + 23:20) - (0 ? + 23:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:20) - (0 ? 23:20) + 1))))))) << (0 ? 23:20))))); + + /* BW */ + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00040, + &totalRead)); + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00044, + &totalWrite)); + + profiler_part2->hi_total_read_8B_count += totalRead; + profiler_part2->hi_total_write_8B_count += totalWrite; + + /* OCB-only BW */ + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_OCB_COUNTER)) + { + if (Hardware->identity.customerID == 0x7e || + Hardware->identity.customerID == 0x7d) + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x17E00, + &totalRead)); + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x17E10, + &totalWrite)); + } + else + { + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x005C0, + &totalRead)); + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x005D0, + &totalWrite)); + + } + } + else + { + totalRead = 0; + totalWrite = 0; + } + + + profiler_part2->hi_total_readOCB_16B_count += totalRead; + profiler_part2->hi_total_writeOCB_16B_count += totalWrite; + + /* PE */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16)))));gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00454, &colorKilled)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16)))));gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00454, &depthKilled)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (2) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16)))));gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00454, &colorDrawn)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16)))));gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00454, &depthDrawn)); + + + if (i == 0) + { + profiler_part1->pe0_pixel_count_killed_by_color_pipe = colorKilled; + profiler_part1->pe0_pixel_count_killed_by_depth_pipe = depthKilled; + profiler_part1->pe0_pixel_count_drawn_by_color_pipe = colorDrawn; + profiler_part1->pe0_pixel_count_drawn_by_depth_pipe = depthDrawn; + } + else if (i == 1) + { + profiler_part1->pe1_pixel_count_killed_by_color_pipe = colorKilled; + profiler_part1->pe1_pixel_count_killed_by_depth_pipe = depthKilled; + profiler_part1->pe1_pixel_count_drawn_by_color_pipe = colorDrawn; + profiler_part1->pe1_pixel_count_drawn_by_depth_pipe = depthDrawn; + } + } + + gcmkUPDATE_PROFILE_DATA_PART2(hi_total_read_8B_count); + gcmkUPDATE_PROFILE_DATA_PART2(hi_total_write_8B_count); + gcmkUPDATE_PROFILE_DATA_PART2(hi_total_readOCB_16B_count); + gcmkUPDATE_PROFILE_DATA_PART2(hi_total_writeOCB_16B_count); +#if USE_SW_RESET + gcmkRESET_PROFILE_DATA_PART1(pe0_pixel_count_killed_by_color_pipe); + gcmkRESET_PROFILE_DATA_PART1(pe0_pixel_count_killed_by_depth_pipe); + gcmkRESET_PROFILE_DATA_PART1(pe0_pixel_count_drawn_by_color_pipe); + gcmkRESET_PROFILE_DATA_PART1(pe0_pixel_count_drawn_by_depth_pipe); + gcmkRESET_PROFILE_DATA_PART1(pe1_pixel_count_killed_by_color_pipe); + gcmkRESET_PROFILE_DATA_PART1(pe1_pixel_count_killed_by_depth_pipe); + gcmkRESET_PROFILE_DATA_PART1(pe1_pixel_count_drawn_by_color_pipe); + gcmkRESET_PROFILE_DATA_PART1(pe1_pixel_count_drawn_by_depth_pipe); +#endif + gcmkUPDATE_PROFILE_DATA_PART1(pe0_pixel_count_killed_by_color_pipe); + gcmkUPDATE_PROFILE_DATA_PART1(pe0_pixel_count_killed_by_depth_pipe); + gcmkUPDATE_PROFILE_DATA_PART1(pe0_pixel_count_drawn_by_color_pipe); + gcmkUPDATE_PROFILE_DATA_PART1(pe0_pixel_count_drawn_by_depth_pipe); + gcmkUPDATE_PROFILE_DATA_PART1(pe1_pixel_count_killed_by_color_pipe); + gcmkUPDATE_PROFILE_DATA_PART1(pe1_pixel_count_killed_by_depth_pipe); + gcmkUPDATE_PROFILE_DATA_PART1(pe1_pixel_count_drawn_by_color_pipe); + gcmkUPDATE_PROFILE_DATA_PART1(pe1_pixel_count_drawn_by_depth_pipe); + + /* Reset clock control register. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + clock)); + + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x0007C, 0)); + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00438, 0)); + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00078, 0)); + +#if !USE_SW_RESET + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) +)); +#endif + + /* FE */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (10) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00450, &profiler_part1->fe_draw_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (11) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00450, &profiler_part1->fe_out_vertex_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (12) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00450, &profiler_part1->fe_cache_miss_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (16) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00450, &profiler_part1->fe_cache_lk_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (17) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00450, &profiler_part1->fe_stall_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (18) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00450, &profiler_part1->fe_process_count)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) +)); + + gcmkUPDATE_PROFILE_DATA_PART1(fe_draw_count); + gcmkUPDATE_PROFILE_DATA_PART1(fe_out_vertex_count); + gcmkUPDATE_PROFILE_DATA_PART1(fe_cache_miss_count); + gcmkUPDATE_PROFILE_DATA_PART1(fe_cache_lk_count); + gcmkUPDATE_PROFILE_DATA_PART1(fe_process_count); + + /* SH */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (7) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->ps_inst_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (8) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->ps_rendered_pixel_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (9) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->vs_inst_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (10) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->vs_rendered_vertice_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (11) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->vs_branch_inst_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (12) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->vs_texld_inst_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (13) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->ps_branch_inst_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (14) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->ps_texld_inst_counter)); + if (newCounters0) + { + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (19) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->vs_non_idle_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (15) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->vs_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (16) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->vs_stall_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (21) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->vs_process_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (20) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->ps_non_idle_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (17) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->ps_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (18) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->ps_stall_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (22) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->ps_process_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->shader_cycle_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (23) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->tx_non_idle_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (24) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->tx_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (25) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->tx_stall_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (26) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &profiler_part1->tx_process_count)); + } +#if USE_SW_RESET + gcmkRESET_PROFILE_DATA_PART1(ps_inst_counter); + gcmkRESET_PROFILE_DATA_PART1(ps_rendered_pixel_counter); + gcmkRESET_PROFILE_DATA_PART1(vs_inst_counter); + gcmkRESET_PROFILE_DATA_PART1(vs_rendered_vertice_counter); + gcmkRESET_PROFILE_DATA_PART1(vs_branch_inst_counter); + gcmkRESET_PROFILE_DATA_PART1(vs_texld_inst_counter); + gcmkRESET_PROFILE_DATA_PART1(ps_branch_inst_counter); + gcmkRESET_PROFILE_DATA_PART1(ps_texld_inst_counter); + if (newCounters0) + { + gcmkRESET_PROFILE_DATA_PART1(vs_non_idle_starve_count); + gcmkRESET_PROFILE_DATA_PART1(vs_starve_count); + gcmkRESET_PROFILE_DATA_PART1(vs_stall_count); + gcmkRESET_PROFILE_DATA_PART1(vs_process_count); + gcmkRESET_PROFILE_DATA_PART1(ps_non_idle_starve_count); + gcmkRESET_PROFILE_DATA_PART1(ps_starve_count); + gcmkRESET_PROFILE_DATA_PART1(ps_stall_count); + gcmkRESET_PROFILE_DATA_PART1(ps_process_count); + gcmkRESET_PROFILE_DATA_PART1(shader_cycle_count); + gcmkRESET_PROFILE_DATA_PART1(tx_non_idle_starve_count); + gcmkRESET_PROFILE_DATA_PART1(tx_starve_count); + gcmkRESET_PROFILE_DATA_PART1(tx_stall_count); + gcmkRESET_PROFILE_DATA_PART1(tx_process_count); + } +#endif + gcmkUPDATE_PROFILE_DATA_PART1(ps_inst_counter); + gcmkUPDATE_PROFILE_DATA_PART1(ps_rendered_pixel_counter); + gcmkUPDATE_PROFILE_DATA_PART1(vs_inst_counter); + gcmkUPDATE_PROFILE_DATA_PART1(vs_rendered_vertice_counter); + gcmkUPDATE_PROFILE_DATA_PART1(vs_branch_inst_counter); + gcmkUPDATE_PROFILE_DATA_PART1(vs_texld_inst_counter); + gcmkUPDATE_PROFILE_DATA_PART1(ps_branch_inst_counter); + gcmkUPDATE_PROFILE_DATA_PART1(ps_texld_inst_counter); + if (newCounters0) + { + gcmkUPDATE_PROFILE_DATA_PART1(vs_non_idle_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(vs_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(vs_stall_count); + gcmkUPDATE_PROFILE_DATA_PART1(vs_process_count); + gcmkUPDATE_PROFILE_DATA_PART1(ps_non_idle_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(ps_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(ps_stall_count); + gcmkUPDATE_PROFILE_DATA_PART1(ps_process_count); + gcmkUPDATE_PROFILE_DATA_PART1(shader_cycle_count); + gcmkUPDATE_PROFILE_DATA_PART1(tx_non_idle_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(tx_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(tx_stall_count); + gcmkUPDATE_PROFILE_DATA_PART1(tx_process_count); + } +#if !USE_SW_RESET + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) +)); +#endif + + /* PA */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_input_vtx_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_input_prim_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (5) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_output_prim_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (6) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_depth_clipped_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (7) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_trivial_rejected_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (8) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_culled_prim_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (9) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_droped_prim_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (10) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_frustum_clipped_prim_counter)); + if (newCounters0) + { + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (12) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_non_idle_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (13) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (14) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_stall_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (15) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00460, &profiler_part1->pa_process_count)); + } +#if USE_SW_RESET + gcmkRESET_PROFILE_DATA_PART1(pa_input_vtx_counter); + gcmkRESET_PROFILE_DATA_PART1(pa_input_prim_counter); + gcmkRESET_PROFILE_DATA_PART1(pa_output_prim_counter); + gcmkRESET_PROFILE_DATA_PART1(pa_depth_clipped_counter); + gcmkRESET_PROFILE_DATA_PART1(pa_trivial_rejected_counter); + gcmkRESET_PROFILE_DATA_PART1(pa_culled_prim_counter); + gcmkRESET_PROFILE_DATA_PART1(pa_droped_prim_counter); + gcmkRESET_PROFILE_DATA_PART1(pa_frustum_clipped_prim_counter); + if (newCounters0) + { + gcmkRESET_PROFILE_DATA_PART1(pa_non_idle_starve_count); + gcmkRESET_PROFILE_DATA_PART1(pa_starve_count); + gcmkRESET_PROFILE_DATA_PART1(pa_stall_count); + gcmkRESET_PROFILE_DATA_PART1(pa_process_count); + } +#endif + gcmkUPDATE_PROFILE_DATA_PART1(pa_input_vtx_counter); + gcmkUPDATE_PROFILE_DATA_PART1(pa_input_prim_counter); + gcmkUPDATE_PROFILE_DATA_PART1(pa_output_prim_counter); + gcmkUPDATE_PROFILE_DATA_PART1(pa_depth_clipped_counter); + gcmkUPDATE_PROFILE_DATA_PART1(pa_trivial_rejected_counter); + gcmkUPDATE_PROFILE_DATA_PART1(pa_culled_prim_counter); + gcmkUPDATE_PROFILE_DATA_PART1(pa_droped_prim_counter); + gcmkUPDATE_PROFILE_DATA_PART1(pa_frustum_clipped_prim_counter); + if (newCounters0) + { + gcmkUPDATE_PROFILE_DATA_PART1(pa_non_idle_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(pa_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(pa_stall_count); + gcmkUPDATE_PROFILE_DATA_PART1(pa_process_count); + } +#if !USE_SW_RESET + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) +)); +#endif + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (15) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_clipped_triangle_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (16) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_clipped_line_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (17) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_culled_triangle_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (18) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_culled_lines_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (19) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_trivial_rejected_line_count)); + if (newCounters0) + { + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (8) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (9) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_stall_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (10) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_receive_triangle_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (11) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_send_triangle_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (12) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_receive_lines_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (13) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_send_lines_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (14) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_process_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (20) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00464, &profiler_part1->se_non_idle_starve_count)); + } + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) +)); + + gcmkUPDATE_PROFILE_DATA_PART1(se_clipped_triangle_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_clipped_line_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_culled_triangle_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_culled_lines_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_trivial_rejected_line_count); + if (newCounters0) + { + gcmkUPDATE_PROFILE_DATA_PART1(se_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_stall_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_receive_triangle_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_send_triangle_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_receive_lines_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_send_lines_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_process_count); + gcmkUPDATE_PROFILE_DATA_PART1(se_non_idle_starve_count); + } + + /* RA */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_valid_pixel_count_to_render)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_total_quad_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (2) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_valid_quad_count_after_early_z)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_input_prim_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (9) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_pipe_cache_miss_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (10) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_prefetch_cache_miss_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (11) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_eez_culled_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (17) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_pipe_hz_cache_miss_counter)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (18) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_prefetch_hz_cache_miss_counter)); + if (newCounters0) + { + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (13) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_non_idle_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (14) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_starve_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (15) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_stall_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (16) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00448, &profiler_part1->ra_process_count)); + } +#if USE_SW_RESET + gcmkRESET_PROFILE_DATA_PART1(ra_valid_pixel_count_to_render); + gcmkRESET_PROFILE_DATA_PART1(ra_total_quad_count); + gcmkRESET_PROFILE_DATA_PART1(ra_valid_quad_count_after_early_z); + gcmkRESET_PROFILE_DATA_PART1(ra_input_prim_count); + gcmkRESET_PROFILE_DATA_PART1(ra_pipe_cache_miss_counter); + gcmkRESET_PROFILE_DATA_PART1(ra_prefetch_cache_miss_counter); + gcmkRESET_PROFILE_DATA_PART1(ra_eez_culled_counter); + gcmkRESET_PROFILE_DATA_PART1(ra_pipe_hz_cache_miss_counter); + gcmkRESET_PROFILE_DATA_PART1(ra_prefetch_hz_cache_miss_counter); + if (newCounters0) + { + gcmkRESET_PROFILE_DATA_PART1(ra_non_idle_starve_count); + gcmkRESET_PROFILE_DATA_PART1(ra_starve_count); + gcmkRESET_PROFILE_DATA_PART1(ra_stall_count); + gcmkRESET_PROFILE_DATA_PART1(ra_process_count); + } +#endif + gcmkUPDATE_PROFILE_DATA_PART1(ra_valid_pixel_count_to_render); + gcmkUPDATE_PROFILE_DATA_PART1(ra_total_quad_count); + gcmkUPDATE_PROFILE_DATA_PART1(ra_valid_quad_count_after_early_z); + gcmkUPDATE_PROFILE_DATA_PART1(ra_input_prim_count); + gcmkUPDATE_PROFILE_DATA_PART1(ra_pipe_cache_miss_counter); + gcmkUPDATE_PROFILE_DATA_PART1(ra_prefetch_cache_miss_counter); + gcmkUPDATE_PROFILE_DATA_PART1(ra_eez_culled_counter); + gcmkUPDATE_PROFILE_DATA_PART1(ra_pipe_hz_cache_miss_counter); + gcmkUPDATE_PROFILE_DATA_PART1(ra_prefetch_hz_cache_miss_counter); + if (newCounters0) + { + gcmkUPDATE_PROFILE_DATA_PART1(ra_non_idle_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(ra_starve_count); + gcmkUPDATE_PROFILE_DATA_PART1(ra_stall_count); + gcmkUPDATE_PROFILE_DATA_PART1(ra_process_count); + } +#if !USE_SW_RESET + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) +)); +#endif + + /* TX */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0044C, &profiler_part1->tx_total_bilinear_requests)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0044C, &profiler_part1->tx_total_trilinear_requests)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (2) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0044C, &profiler_part1->tx_total_discarded_texture_requests)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0044C, &profiler_part1->tx_total_texture_requests)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (5) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0044C, &profiler_part1->tx_mc0_miss_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (6) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0044C, &profiler_part1->tx_mc0_request_byte_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (7) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0044C, &profiler_part1->tx_mc1_miss_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (8) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0044C, &profiler_part1->tx_mc1_request_byte_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00474, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) +)); + + gcmkUPDATE_PROFILE_DATA_PART1(tx_total_bilinear_requests); + gcmkUPDATE_PROFILE_DATA_PART1(tx_total_trilinear_requests); + gcmkUPDATE_PROFILE_DATA_PART1(tx_total_discarded_texture_requests); + gcmkUPDATE_PROFILE_DATA_PART1(tx_total_texture_requests); + gcmkUPDATE_PROFILE_DATA_PART1(tx_mc0_miss_count); + gcmkUPDATE_PROFILE_DATA_PART1(tx_mc0_request_byte_count); + gcmkUPDATE_PROFILE_DATA_PART1(tx_mc1_miss_count); + gcmkUPDATE_PROFILE_DATA_PART1(tx_mc1_request_byte_count); + + /* MC */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_read_req_8B_from_colorpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (2) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_read_req_8B_sentout_from_colorpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_write_req_8B_from_colorpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_read_req_sentout_from_colorpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (5) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_write_req_from_colorpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (7) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_read_req_8B_from_depthpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (8) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_read_req_8B_sentout_from_depthpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (9) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_write_req_8B_from_depthpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (10) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_read_req_sentout_from_depthpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (11) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_write_req_from_depthpipe)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (12) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_read_req_8B_from_others)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (13) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_write_req_8B_from_others)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (14) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_read_req_from_others)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (15) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mcc_total_write_req_from_others)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (21) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mc_fe_read_bandwidth)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (22) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mc_mmu_read_bandwidth)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (23) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mc_blt_read_bandwidth)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (24) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mc_sh0_read_bandwidth)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (25) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mc_sh1_read_bandwidth)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (26) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mc_pe_write_bandwidth)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (27) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mc_blt_write_bandwidth)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (28) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mc_sh0_write_bandwidth)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (29) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00468, &profiler_part2->mc_sh1_write_bandwidth)); + + /* Reset counters. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x0003C, 1)); + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x0003C, 0)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) +)); + + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_read_req_8B_from_colorpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_read_req_8B_sentout_from_colorpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_write_req_8B_from_colorpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_read_req_sentout_from_colorpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_write_req_from_colorpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_read_req_8B_from_depthpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_read_req_8B_sentout_from_depthpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_write_req_8B_from_depthpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_read_req_sentout_from_depthpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_write_req_from_depthpipe); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_read_req_8B_from_others); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_write_req_8B_from_others); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_read_req_from_others); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_total_write_req_from_others); + gcmkUPDATE_PROFILE_DATA_PART2(mc_fe_read_bandwidth); + gcmkUPDATE_PROFILE_DATA_PART2(mc_mmu_read_bandwidth); + gcmkUPDATE_PROFILE_DATA_PART2(mc_blt_read_bandwidth); + gcmkUPDATE_PROFILE_DATA_PART2(mc_sh0_read_bandwidth); + gcmkUPDATE_PROFILE_DATA_PART2(mc_sh1_read_bandwidth); + gcmkUPDATE_PROFILE_DATA_PART2(mc_pe_write_bandwidth); + gcmkUPDATE_PROFILE_DATA_PART2(mc_blt_write_bandwidth); + gcmkUPDATE_PROFILE_DATA_PART2(mc_sh0_write_bandwidth); + gcmkUPDATE_PROFILE_DATA_PART2(mc_sh1_write_bandwidth); + + /* read latency counters */ + if (newCounters0) + { + /* latency */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x0056C, + &mc_axi_max_min_latency)); + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00570, + &profiler_part2->mcc_axi_total_latency)); + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00574, + &profiler_part2->mcc_axi_sample_count)); + + /* Reset Latency counters */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00568, + 0x10a)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00568, + 0xa)); + + profiler_part2->mcc_axi_min_latency = (mc_axi_max_min_latency & 0xffff0000) >> 16; + profiler_part2->mcc_axi_max_latency = (mc_axi_max_min_latency & 0x0000ffff); + if (profiler_part2->mcc_axi_min_latency == 4095) + profiler_part2->mcc_axi_min_latency = 0; + + gcmkUPDATE_PROFILE_DATA_PART2(mcc_axi_min_latency); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_axi_max_latency); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_axi_total_latency); + gcmkUPDATE_PROFILE_DATA_PART2(mcc_axi_sample_count); + } + + /* HI */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0046C, &profiler_part2->hi0_axi_cycles_read_request_stalled)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0046C, &profiler_part2->hi0_axi_cycles_write_request_stalled)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (2) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0046C, &profiler_part2->hi0_axi_cycles_write_data_stalled)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))) +)); + + gcmkUPDATE_PROFILE_DATA_PART2(hi0_axi_cycles_read_request_stalled); + gcmkUPDATE_PROFILE_DATA_PART2(hi0_axi_cycles_write_request_stalled); + gcmkUPDATE_PROFILE_DATA_PART2(hi0_axi_cycles_write_data_stalled); + + /* L2 */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_total_axi0_read_request_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_total_axi0_write_request_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (5) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_total_axi1_write_request_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (8) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_total_read_transactions_request_by_axi0)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (9) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_total_read_transactions_request_by_axi1)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (12) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_total_write_transactions_request_by_axi0)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (13) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_total_write_transactions_request_by_axi1)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (16) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_axi0_minmax_latency)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (17) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_axi0_total_latency)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (18) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_axi0_total_request_count)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (19) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_axi1_minmax_latency)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (20) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_axi1_total_latency)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (21) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00564, &profiler_part2->l2_axi1_total_request_count)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (resetValue) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) )); +gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00478, ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) +)); + + profiler_part2->l2_axi0_min_latency = (profiler_part2->l2_axi0_minmax_latency & 0xffff0000) >> 16; + profiler_part2->l2_axi0_max_latency = (profiler_part2->l2_axi0_minmax_latency & 0x0000ffff); + profiler_part2->l2_axi1_min_latency = (profiler_part2->l2_axi0_minmax_latency & 0xffff0000) >> 16; + profiler_part2->l2_axi1_max_latency = (profiler_part2->l2_axi0_minmax_latency & 0x0000ffff); + gcmkUPDATE_PROFILE_DATA_PART2(l2_total_axi0_read_request_count); + gcmkUPDATE_PROFILE_DATA_PART2(l2_total_axi1_read_request_count); + gcmkUPDATE_PROFILE_DATA_PART2(l2_total_axi0_write_request_count); + gcmkUPDATE_PROFILE_DATA_PART2(l2_total_axi1_write_request_count); + gcmkUPDATE_PROFILE_DATA_PART2(l2_total_read_transactions_request_by_axi0); + gcmkUPDATE_PROFILE_DATA_PART2(l2_total_read_transactions_request_by_axi1); + gcmkUPDATE_PROFILE_DATA_PART2(l2_total_write_transactions_request_by_axi0); + gcmkUPDATE_PROFILE_DATA_PART2(l2_total_write_transactions_request_by_axi1); + gcmkUPDATE_PROFILE_DATA_PART2(l2_axi0_min_latency); + gcmkUPDATE_PROFILE_DATA_PART2(l2_axi0_max_latency); + gcmkUPDATE_PROFILE_DATA_PART2(l2_axi0_total_latency); + gcmkUPDATE_PROFILE_DATA_PART2(l2_axi0_total_request_count); + gcmkUPDATE_PROFILE_DATA_PART2(l2_axi1_min_latency); + gcmkUPDATE_PROFILE_DATA_PART2(l2_axi1_max_latency); + gcmkUPDATE_PROFILE_DATA_PART2(l2_axi1_total_latency); + gcmkUPDATE_PROFILE_DATA_PART2(l2_axi1_total_request_count); + + gcmkVERIFY_OK(gckOS_ReleaseMutex( + command->os, command->mutexContextSeq + )); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (mutexAcquired) + { + gckOS_ReleaseMutex( + command->os, command->mutexContextSeq + ); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + + +gceSTATUS +gckHARDWARE_InitProfiler( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gctUINT32 control; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + &control)); + /* Enable debug register. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + ((((gctUINT32) (control)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 11:11) - (0 ? + 11:11) + 1))))))) << (0 ? + 11:11))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 11:11) - (0 ? + 11:11) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 11:11) - (0 ? 11:11) + 1))))))) << (0 ? 11:11))))); + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_ResetGPU( + IN gckHARDWARE Hardware, + IN gckOS Os, + IN gceCORE Core + ) +{ + gctUINT32 control, idle; + gceSTATUS status; + gctUINT32 count = 0; + gctUINT32 mmuEnabled; + + while (count < 2) + { + /* Disable clock gating. */ + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + Hardware->powerBaseAddress + + 0x00104, + 0x00000000)); + + control = ((((gctUINT32) (0x01590880)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:17) - (0 ? + 17:17) + 1))))))) << (0 ? + 17:17))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:17) - (0 ? 17:17) + 1))))))) << (0 ? 17:17))); + + /* Disable pulse-eater. */ + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x0010C, + control)); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x0010C, + ((((gctUINT32) (control)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))))); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x0010C, + control)); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x00000, + ((((gctUINT32) (0x00000900)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:9) - (0 ? + 9:9) + 1))))))) << (0 ? + 9:9))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 9:9) - (0 ? + 9:9) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:9) - (0 ? 9:9) + 1))))))) << (0 ? 9:9))))); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x00000, + 0x00000900)); + + /* Wait for clock being stable. */ + gcmkONERROR(gckOS_Delay(Os, 1)); + + /* Isolate the GPU. */ + control = ((((gctUINT32) (0x00000900)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 19:19) - (0 ? + 19:19) + 1))))))) << (0 ? + 19:19))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 19:19) - (0 ? 19:19) + 1))))))) << (0 ? 19:19))); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x00000, + control)); + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_SECURITY_AHB) && + (Hardware->options.secureMode == gcvSECURE_IN_NORMAL)) + { + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x003A8, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))))); + } + else + { + /* Set soft reset. */ + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x00000, + ((((gctUINT32) (control)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:12) - (0 ? + 12:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:12) - (0 ? + 12:12) + 1))))))) << (0 ? + 12:12))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 12:12) - (0 ? + 12:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:12) - (0 ? 12:12) + 1))))))) << (0 ? 12:12))))); + } + +#if gcdFPGA_BUILD + /* Wait more time on FPGA for reset as lower frequency */ + gcmkONERROR(gckOS_Delay(Os, 10)); +#else + /* Wait for reset. */ + gcmkONERROR(gckOS_Delay(Os, 1)); +#endif + /* Reset soft reset bit. */ + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x00000, + ((((gctUINT32) (control)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:12) - (0 ? + 12:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:12) - (0 ? + 12:12) + 1))))))) << (0 ? + 12:12))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 12:12) - (0 ? + 12:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:12) - (0 ? 12:12) + 1))))))) << (0 ? 12:12))))); + + /* Reset GPU isolation. */ + control = ((((gctUINT32) (control)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 19:19) - (0 ? + 19:19) + 1))))))) << (0 ? + 19:19))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 19:19) - (0 ? 19:19) + 1))))))) << (0 ? 19:19))); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, + Core, + 0x00000, + control)); + + /* Read idle register. */ + gcmkONERROR(gckOS_ReadRegisterEx(Os, + Core, + 0x00004, + &idle)); + + + if ((((((gctUINT32) (idle)) >> (0 ? 0:0)) & ((gctUINT32) ((((1 ? 0:0) - (0 ? 0:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1)))))) ) == 0) + { + continue; + } + + gcmkDUMP(Os, "@[register.wait 0x%05X 0x%08X 0x%08X]", + 0x00004, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (~0U) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))), + idle); + + /* Read reset register. */ + gcmkONERROR(gckOS_ReadRegisterEx(Os, + Core, + 0x00000, + &control)); + if (((((((gctUINT32) (control)) >> (0 ? 16:16)) & ((gctUINT32) ((((1 ? 16:16) - (0 ? 16:16) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1)))))) ) == 0) + || ((((((gctUINT32) (control)) >> (0 ? 17:17)) & ((gctUINT32) ((((1 ? 17:17) - (0 ? 17:17) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 17:17) - (0 ? 17:17) + 1)))))) ) == 0) + ) + { + continue; + } + + gcmkDUMP(Os, "@[register.wait 0x%05X 0x%08X 0x%08X]", + 0x00000, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) ((gctUINT32) (~0U) & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:17) - (0 ? + 17:17) + 1))))))) << (0 ? + 17:17))) | (((gctUINT32) ((gctUINT32) (~0U) & ((gctUINT32) ((((1 ? + 17:17) - (0 ? + 17:17) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:17) - (0 ? 17:17) + 1))))))) << (0 ? 17:17))), + control); + + /* Force Disable MMU to guarantee setup command be read from physical addr */ + if (Hardware->options.secureMode == gcvSECURE_IN_NORMAL) + { + gctUINT32 regMmuCtrl = 0; + gcmkONERROR(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x00388, + ®MmuCtrl + )); + + mmuEnabled = (((((gctUINT32) (regMmuCtrl)) >> (0 ? 0:0)) & ((gctUINT32) ((((1 ? 0:0) - (0 ? 0:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1)))))) ); + } + else + { + gctUINT32 regMmuCtrl = 0; + + gcmkONERROR(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x0018C, + ®MmuCtrl + )); + + mmuEnabled = (((((gctUINT32) (regMmuCtrl)) >> (0 ? 0:0)) & ((gctUINT32) ((((1 ? 0:0) - (0 ? 0:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1)))))) ); + } + + if (mmuEnabled) + { + /* Not reset properly, reset again. */ + continue; + } + + count++; + } + + /* Success. */ + return gcvSTATUS_OK; + +OnError: + + /* Return the error. */ + return status; +} + +gceSTATUS +gckHARDWARE_Reset( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_OBJECT(Hardware->kernel, gcvOBJ_KERNEL); + + /* Record context ID in debug register before reset. */ + gcmkONERROR(gckHARDWARE_UpdateContextID(Hardware)); + + /* Hardware reset. */ + status = gckOS_ResetGPU(Hardware->os, Hardware->core); + + if (gcmIS_ERROR(status)) + { + if (Hardware->identity.chipRevision < 0x4600) + { + /* Not supported - we need the isolation bit. */ + gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); + } + + /* Soft reset. */ + gcmkONERROR(_ResetGPU(Hardware, Hardware->os, Hardware->core)); + } + + /* Force the command queue to reload the next context. */ + Hardware->kernel->command->currContext = gcvNULL; + + /* Initialize hardware. */ + gcmkONERROR(gckHARDWARE_InitializeHardware(Hardware)); + + /* Jump to address into which GPU should run if it doesn't stuck. */ + if (Hardware->wlFE) + { + gcmkONERROR(gckWLFE_Execute(Hardware, Hardware->kernel->restoreAddress, 16)); + } + + gcmkPRINT("[galcore]: recovery done"); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkPRINT("[galcore]: Hardware not reset successfully, give up"); + + /* Return the error. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_GetBaseAddress( + IN gckHARDWARE Hardware, + OUT gctUINT32_PTR BaseAddress + ) +{ + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(BaseAddress != gcvNULL); + + /* Get the base address from the OS. */ + *BaseAddress = Hardware->baseAddress; + + /* Success. */ + gcmkFOOTER_ARG("*BaseAddress=0x%08x", *BaseAddress); + return gcvSTATUS_OK; +} + +gceSTATUS +gckHARDWARE_NeedBaseAddress( + IN gckHARDWARE Hardware, + IN gctUINT32 State, + OUT gctBOOL_PTR NeedBase + ) +{ + gctBOOL need = gcvFALSE; + + gcmkHEADER_ARG("Hardware=0x%x State=0x%08x", Hardware, State); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(NeedBase != gcvNULL); + + /* Make sure this is a load state. */ + if (((((gctUINT32) (State)) >> (0 ? + 31:27) & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1)))))) == (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1)))))))) + { +#if gcdENABLE_3D + /* Get the state address. */ + switch ((((((gctUINT32) (State)) >> (0 ? 15:0)) & ((gctUINT32) ((((1 ? 15:0) - (0 ? 15:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1)))))) )) + { + case 0x0596: + case 0x0597: + case 0x0599: + case 0x059A: + case 0x05A9: + /* These states need a TRUE physical address if MC20 fix is not there */ + need = (gcvSTATUS_FALSE == gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_MC20)); + break; + } +#else + /* 2D addresses don't need a base address. */ +#endif + } + + /* Return the flag. */ + *NeedBase = need; + + /* Success. */ + gcmkFOOTER_ARG("*NeedBase=%d", *NeedBase); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHARDWARE_IsFeatureAvailable +** +** Verifies whether the specified feature is available in hardware. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** gceFEATURE Feature +** Feature to be verified. +*/ +gceSTATUS +gckHARDWARE_IsFeatureAvailable( + IN gckHARDWARE Hardware, + IN gceFEATURE Feature + ) +{ + gctBOOL available; + + gcmkHEADER_ARG("Hardware=0x%x Feature=%d", Hardware, Feature); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + available = _QueryFeatureDatabase(Hardware, Feature); + + /* Return result. */ + gcmkFOOTER_ARG("%d", available ? gcvSTATUS_TRUE : gcvSTATUS_FALSE); + return available ? gcvSTATUS_TRUE : gcvSTATUS_FALSE; +} + +gceSTATUS +gckHARDWARE_QueryMcfe( + IN gckHARDWARE Hardware, + OUT const gceMCFE_CHANNEL_TYPE * Channels[], + OUT gctUINT32 * Count + ) +{ + if (!_QueryFeatureDatabase(Hardware, gcvFEATURE_MCFE)) + { + /* No MCFE feature. */ + return gcvSTATUS_NOT_SUPPORTED; + } + + if (Channels) + { + *Channels = Hardware->mcfeChannels; + } + + if (Count) + { + *Count = Hardware->mcfeChannelCount; + } + + return gcvSTATUS_OK; +} + + +/******************************************************************************* +** +** gckHARDWARE_DumpMMUException +** +** Dump the MMU debug info on an MMU exception. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_DumpMMUException( + IN gckHARDWARE Hardware + ) +{ + gctUINT32 mmu = 0; + gctUINT32 mmuStatus = 0; + gctUINT32 address = 0; + gctUINT32 i = 0; + gctUINT32 mtlb = 0; + gctUINT32 stlb = 0; + gctUINT32 offset = 0; + gctUINT32 mmuStatusRegAddress; + gctUINT32 mmuExceptionAddress; + gceAREA_TYPE areaType = gcvAREA_TYPE_UNKNOWN; + gctUINT32 stlbShift; + gctUINT32 stlbMask; + gctUINT32 pgoffMask; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + +#if gcdENABLE_TRUST_APPLICATION + if (Hardware->options.secureMode == gcvSECURE_IN_TA) + { + gcmkVERIFY_OK(gckKERNEL_SecurityDumpMMUException(Hardware->kernel)); + + gckMMU_DumpRecentFreedAddress(Hardware->kernel->mmu); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + else +#endif + if (Hardware->options.secureMode == gcvSECURE_NONE) + { + mmuStatusRegAddress = 0x00188; + mmuExceptionAddress = 0x00190; + } + else + { + mmuStatusRegAddress = 0x00384; + mmuExceptionAddress = 0x00380; + } + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + gcmkPRINT("GPU[%d](ChipModel=0x%x ChipRevision=0x%x):\n", + Hardware->core, + Hardware->identity.chipModel, + Hardware->identity.chipRevision); + + gcmkPRINT("**************************\n"); + gcmkPRINT("*** MMU ERROR DUMP ***\n"); + gcmkPRINT("**************************\n"); + + + gcmkVERIFY_OK( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + mmuStatusRegAddress, + &mmuStatus)); + + gcmkPRINT(" MMU status = 0x%08X\n", mmuStatus); + + for (i = 0; i < 4; i += 1) + { + mmu = mmuStatus & 0xF; + mmuStatus >>= 4; + + if (mmu == 0) + { + continue; + } + + switch (mmu) + { + case 1: + gcmkPRINT(" MMU%d: slave not present\n", i); + break; + + case 2: + gcmkPRINT(" MMU%d: page not present\n", i); + break; + + case 3: + gcmkPRINT(" MMU%d: write violation\n", i); + break; + + case 4: + gcmkPRINT(" MMU%d: out of bound", i); + break; + + case 5: + gcmkPRINT(" MMU%d: read security violation", i); + break; + + case 6: + gcmkPRINT(" MMU%d: write security violation", i); + break; + + default: + gcmkPRINT(" MMU%d: unknown state\n", i); + } + + if (Hardware->options.secureMode == gcvSECURE_NONE) + { + gcmkVERIFY_OK( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + mmuExceptionAddress + i * 4, + &address)); + } + else + { + gcmkVERIFY_OK( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + mmuExceptionAddress, + &address)); + } + + gckMMU_GetAreaType(Hardware->kernel->mmu, address, &areaType); + + if (areaType == gcvAREA_TYPE_UNKNOWN) + { + gcmkPRINT(" MMU%d: exception address = 0x%08X, it is not mapped.\n", i, address); + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + pgoffMask = (areaType == gcvAREA_TYPE_4K) ? gcdMMU_OFFSET_4K_MASK : gcdMMU_OFFSET_1M_MASK; + stlbShift = (areaType == gcvAREA_TYPE_4K) ? gcdMMU_STLB_4K_SHIFT : gcdMMU_STLB_1M_SHIFT; + stlbMask = (areaType == gcvAREA_TYPE_4K) ? gcdMMU_STLB_4K_MASK : gcdMMU_STLB_1M_MASK; + + mtlb = (address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; + stlb = (address & stlbMask) >> stlbShift; + offset = address & pgoffMask; + + gcmkPRINT(" MMU%d: exception address = 0x%08X\n", i, address); + + gcmkPRINT(" MTLB entry = %d\n", mtlb); + + gcmkPRINT(" STLB entry = %d\n", stlb); + + gcmkPRINT(" Offset = 0x%08X (%d)\n", offset, offset); + + gckMMU_DumpPageTableEntry(Hardware->kernel->mmu, areaType, address); + + gckMMU_DumpRecentFreedAddress(Hardware->kernel->mmu); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckHARDWARE_HandleFault( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status = gcvSTATUS_NOT_SUPPORTED; + gctUINT32 mmu, mmuStatus, address = gcvINVALID_ADDRESS, i = 0; + gctUINT32 mmuStatusRegAddress; + gctUINT32 mmuExceptionAddress; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + if (Hardware->options.secureMode == gcvSECURE_NONE) + { + mmuStatusRegAddress = 0x00188; + mmuExceptionAddress = 0x00190; + } + else + { + mmuStatusRegAddress = 0x00384; + mmuExceptionAddress = 0x00380; + } + + /* Get MMU exception address. */ +#if gcdENABLE_TRUST_APPLICATION + if (Hardware->options.secureMode == gcvSECURE_IN_TA) + { + gckKERNEL_ReadMMUException(Hardware->kernel, &mmuStatus, &address); + } + else +#endif + { + gcmkVERIFY_OK(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + mmuStatusRegAddress, + &mmuStatus + )); + + gcmkPRINT(" MMU status = 0x%08X\n", mmuStatus); + + for (i = 0; i < 4; i += 1) + { + mmu = mmuStatus & 0xF; + mmuStatus >>= 4; + + if (mmu == 0) + { + continue; + } + + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + mmuExceptionAddress + i * 4, + &address + )); + + break; + } + } + + if (address != gcvINVALID_ADDRESS) + { + gckVIDMEM_NODE nodeObject = gcvNULL; + gctUINT32 offset = 0; + gctPHYS_ADDR_T physicalAddress = 0; + gceAREA_TYPE areaType; + gctUINT32 pageMask; + gcePAGE_TYPE pageType; + + gckMMU_GetAreaType(Hardware->kernel->mmu, address, &areaType); + + pageMask = (areaType == gcvAREA_TYPE_4K) ? gcdMMU_PAGE_4K_MASK : gcdMMU_PAGE_1M_MASK; + pageType = (areaType == gcvAREA_TYPE_4K) ? gcvPAGE_TYPE_4K : gcvPAGE_TYPE_1M; + +#if gcdENABLE_TRUST_APPLICATION + address &= ~gcdMMU_PAGE_4K_MASK; +#else + address &= ~pageMask; +#endif + + /* Try to allocate memory and setup map for exception address. */ + gcmkONERROR(gckVIDMEM_NODE_Find( + Hardware->kernel, + address, + &nodeObject, + &offset + )); + + gcmkONERROR(gckVIDMEM_NODE_GetPhysical( + Hardware->kernel, + nodeObject, + offset, + &physicalAddress + )); + +#if gcdENABLE_TRUST_APPLICATION + if (Hardware->options.secureMode == gcvSECURE_IN_TA) + { + gckKERNEL_HandleMMUException( + Hardware->kernel, + mmuStatus, + physicalAddress, + address + ); + } + else +#endif + { + gctUINT32_PTR entry; + + /* Setup page table. */ + gcmkONERROR(gckMMU_GetPageEntry(Hardware->kernel->mmu, pageType, address, &entry)); + + gckMMU_SetPage(Hardware->kernel->mmu, physicalAddress, pageType, gcvTRUE, entry); + + /* Resume hardware execution. */ + gcmkVERIFY_OK(gckOS_WriteRegisterEx( + Hardware->os, + Hardware->core, + mmuExceptionAddress + i * 4, + *entry + )); + } + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_DumpMCFEState( + IN gckOS Os, + IN gceCORE Core + ) +{ + gctUINT32 i, j, data = 0, array[8] = { 0 }; + gceSTATUS status = gcvSTATUS_OK; + + gcmkHEADER(); + + gcmkPRINT("**************************\n"); + gcmkPRINT("***** MCFE STATE *****\n"); + gcmkPRINT("**************************\n"); + + /* Fetch address of channels. */ + gcmkPRINT("Channel fetch addresses:\n"); + gcmkPRINT(" [00] [01] [02] [03]\n"); + + for (i = 0; i < 4; i++) + { + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x3 + 2 * i)); + } + + for (i = 0; i < 16; i++) + { + for (j = 0; j < 4; j++) + { + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x2 + 2 * j)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &array[j])); + } + + gcmkPRINT(" 0x%08X 0x%08X 0x%08X 0x%08X\n", array[0], array[1], array[2], array[3]); + } + + /* Command data of channels. */ + gcmkPRINT_N(0, "Channel command data:\n"); + gcmkPRINT_N(0, " [00] [01] [02] [03]\n"); + gcmkPRINT_N(0, " [Low High] [Low High] [Low High] [Low High]\n"); + + + for (i = 0; i < 4; i++) + { + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x11 + 4 * i)); + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x11 + 4 * i + 2)); + } + + for (i = 0; i < 32; i++) + { + for (j = 0; j < 4; j++) + { + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x10 + 4 * j)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &array[j * 2])); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x10 + 4 * j + 2)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &array[j * 2 + 1])); + } + + gcmkPRINT_N(0, " 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n", + array[0], array[1], array[2], array[3], array[4], array[5], array[6], array[7]); + } + + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x00)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &data)); + gcmkPRINT_N(0, "0x00: 0x%08X\n", data); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x01)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &data)); + gcmkPRINT_N(0, "0x01: 0x%08X\n", data); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x0A)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &data)); + gcmkPRINT_N(0, "WaitSemaphore: 0x%08X\n", data); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x0B)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &data)); + gcmkPRINT_N(0, "WaitEventID(channel 0 and 1): 0x%08X\n", data); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x0C)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &data)); + gcmkPRINT_N(0, "WaitEventID(channel 2 and 3): 0x%08X\n", data); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x0D)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &data)); + gcmkPRINT_N(0, "DecodeState: 0x%08X\n", data); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x0E)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &data)); + gcmkPRINT_N(0, "DebugSelect(0x0E): 0x%08X\n", data); + + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x0F)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &data)); + gcmkPRINT_N(0, "DebugSelect(0x0F): 0x%08X\n", data); + + for (i = 0; i < 9; i++) + { + gcmkONERROR(gckOS_WriteRegisterEx(Os, Core, 0x470, 0x20 + i)); + gcmkONERROR(gckOS_ReadRegisterEx(Os, Core, 0x450, &data)); + gcmkPRINT_N(0, "DebugSelect(0x%02X): 0x%08X\n", 0x20 + i, data); + } + +OnError: + gcmkFOOTER(); + + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_DumpGPUState +** +** Dump the GPU debug registers. +** +** INPUT: +** +** gckHARDWARE Harwdare +** Pointer to an gckHARDWARE object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHARDWARE_DumpGPUState( + IN gckHARDWARE Hardware + ) +{ + static gctCONST_STRING _cmdState[] = + { + "PAR_IDLE_ST", "PAR_DEC_ST", "PAR_ADR0_ST", "PAR_LOAD0_ST", + "PAR_ADR1_ST", "PAR_LOAD1_ST", "PAR_3DADR_ST", "PAR_3DCMD_ST", + "PAR_3DCNTL_ST", "PAR_3DIDXCNTL_ST", "PAR_INITREQDMA_ST", "PAR_DRAWIDX_ST", + "PAR_DRAW_ST", "PAR_2DRECT0_ST", "PAR_2DRECT1_ST", "PAR_2DDATA0_ST", + "PAR_2DDATA1_ST", "PAR_WAITFIFO_ST", "PAR_WAIT_ST", "PAR_LINK_ST", + "PAR_END_ST", "PAR_STALL_ST", "INVALID_PAR_ST", "INVALID_PAR_ST", + "INVALID_PAR_ST", "INVALID_PAR_ST", "INVALID_PAR_ST", "INVALID_PAR_ST", + "INVALID_PAR_ST", "INVALID_PAR_ST", "INVALID_PAR_ST", "INVALID_PAR_ST" + }; + + static gctCONST_STRING _cmdDmaState[] = + { + "CMD_IDLE_ST", "CMD_START_ST", "CMD_REQ_ST", "CMD_END_ST" + }; + + static gctCONST_STRING _cmdFetState[] = + { + "FET_IDLE_ST", "FET_RAMVALID_ST", "FET_VALID_ST", "INVALID_FET_ST" + }; + + static gctCONST_STRING _reqDmaState[] = + { + "REQ_IDLE_ST", "REQ_WAITIDX_ST", "REQ_CAL_ST", "INVALID_REQ_ST" + }; + + static gctCONST_STRING _calState[] = + { + "CAL_IDLE_ST", "CAL_LDADR_ST", "CAL_IDXCALC_ST", "INVALID_CAL_ST" + }; + + static gctCONST_STRING _veReqState[] = + { + "VER_IDLE_ST", "VER_CKCACHE_ST", "VER_MISS_ST", "INVALID_VER_ST" + }; + + enum + { + RA_INDEX = 0, + TX_INDEX = 1, + FE_INDEX = 2, + PE_INDEX = 3, + DE_INDEX = 4, + SH_INDEX = 5, + PA_INDEX = 6, + SE_INDEX = 7, + MC_INDEX = 8, + HI_INDEX = 9, + TPG_INDEX = 10, + TFB_INDEX = 11, + USC_INDEX = 12, + L2_INDEX = 13, + BLT_INDEX = 14, + WD_INDEX = 15, + VTXDATA_INDEX = 16, + DIR_INDEX = 17, + PPA_INDEX = 18, + NN_INDEX = 19, + MODULE_MAX_INDEX, + }; + + /* must keep order correctly for _dbgRegs, we need ajust some value base on the index */ + static gcsiDEBUG_REGISTERS _dbgRegs[MODULE_MAX_INDEX] = + { + { "RA", 0x474, 16, 0x448, 256, 0x1, 0x00, gcvTRUE, gcvTRUE }, + { "TX", 0x474, 24, 0x44C, 128, 0x1, 0x00, gcvTRUE, gcvTRUE }, + { "FE", 0x470, 0, 0x450, 256, 0x1, 0x00, gcvTRUE, gcvFALSE }, + { "PE", 0x470, 16, 0x454, 256, 0x3, 0x00, gcvTRUE, gcvTRUE }, + { "DE", 0x470, 8, 0x458, 256, 0x1, 0x00, gcvTRUE, gcvFALSE }, + { "SH", 0x470, 24, 0x45C, 256, 0x1, 0x00, gcvTRUE, gcvTRUE }, + { "PA", 0x474, 0, 0x460, 256, 0x1, 0x00, gcvTRUE, gcvTRUE }, + { "SE", 0x474, 8, 0x464, 256, 0x1, 0x00, gcvTRUE, gcvTRUE }, + { "MC", 0x478, 0, 0x468, 256, 0x3, 0x00, gcvTRUE, gcvTRUE }, + { "HI", 0x478, 8, 0x46C, 256, 0x1, 0x00, gcvTRUE, gcvFALSE }, + { "TPG", 0x474, 24, 0x44C, 32, 0x2, 0x80, gcvFALSE, gcvTRUE }, + { "TFB", 0x474, 24, 0x44C, 32, 0x2, 0xA0, gcvFALSE, gcvTRUE }, + { "USC", 0x474, 24, 0x44C, 64, 0x2, 0xC0, gcvFALSE, gcvTRUE }, + { "L2", 0x478, 0, 0x564, 256, 0x1, 0x00, gcvTRUE, gcvFALSE }, + { "BLT", 0x478, 24, 0x1A4, 256, 0x1, 0x00, gcvFALSE, gcvTRUE }, + { "WD", 0xF0, 16, 0xF4, 256, 0x1, 0x00, gcvFALSE, gcvFALSE }, + { "VTXDATA", 0x474, 24, 0x44C, 64, 0x1, 0x40, gcvFALSE, gcvTRUE }, + { "DIR", 0xF0, 24, 0xF8, 256, 0x1, 0x00, gcvFALSE, gcvTRUE }, + { "PPA", 0x474, 0, 0x598, 256, 0x1, 0x00, gcvFALSE, gcvTRUE }, + { "NN", 0x474, 24, 0x44C, 256, 0x2, 0x00, gcvFALSE, gcvTRUE }, + + }; + + static gctUINT32 _otherRegs[] = + { + 0x040, 0x044, 0x04C, 0x050, 0x054, 0x058, 0x05C, 0x060, + 0x43c, 0x440, 0x444, 0x414, 0x100 + }; + + gceSTATUS status; + gctUINT32 idle = 0, axi = 0; + gctUINT32 dmaAddress1 = 0, dmaAddress2 = 0; + gctUINT32 dmaState1 = 0, dmaState2 = 0; + gctUINT32 dmaLow = 0, dmaHigh = 0; + gctUINT32 cmdState = 0, cmdDmaState = 0, cmdFetState = 0; + gctUINT32 dmaReqState = 0, calState = 0, veReqState = 0; + gctUINT i; + gctUINT pipe = 0, pipeMask = 0x1; + static const gctUINT maxNumOfPipes = 4; + gctUINT32 control = 0, oldControl = 0; + gckOS os = Hardware->os; + gceCORE core = Hardware->core; + gceSTATUS hwTFB = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_HW_TFB); + gceSTATUS usc = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_USC); + gceSTATUS multiCluster = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_MULTI_CLUSTER); + gceSTATUS bltEngine = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_BLT_ENGINE); + gceSTATUS gsShader = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_GEOMETRY_SHADER); + gceSTATUS nnEngine = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_NN_ENGINE); + + gcmkHEADER_ARG("Hardware=0x%X", Hardware); + + gcmkPRINT_N(12, "GPU[%d](ChipModel=0x%x ChipRevision=0x%x):\n", + core, + Hardware->identity.chipModel, + Hardware->identity.chipRevision); + + /* Reset register values. */ + idle = axi = + dmaState1 = dmaState2 = + dmaAddress1 = dmaAddress2 = + dmaLow = dmaHigh = 0; + + switch (Hardware->identity.pixelPipes) + { + case 2: + pipeMask = 0x3; + break; + case 1: + pipeMask = 0x1; + break; + default: + gcmkASSERT(0); + } + /* Verify whether DMA is running. */ + gcmkONERROR(_VerifyDMA( + os, core, &dmaAddress1, &dmaAddress2, &dmaState1, &dmaState2 + )); + + cmdState = dmaState2 & 0x1F; + cmdDmaState = (dmaState2 >> 8) & 0x03; + cmdFetState = (dmaState2 >> 10) & 0x03; + dmaReqState = (dmaState2 >> 12) & 0x03; + calState = (dmaState2 >> 14) & 0x03; + veReqState = (dmaState2 >> 16) & 0x03; + + gcmkONERROR(gckOS_ReadRegisterEx(os, core, 0x00004, &idle)); + gcmkONERROR(gckOS_ReadRegisterEx(os, core, 0x0000C, &axi)); + gcmkONERROR(gckOS_ReadRegisterEx(os, core, 0x00668, &dmaLow)); + gcmkONERROR(gckOS_ReadRegisterEx(os, core, 0x00668, &dmaLow)); + gcmkONERROR(gckOS_ReadRegisterEx(os, core, 0x0066C, &dmaHigh)); + gcmkONERROR(gckOS_ReadRegisterEx(os, core, 0x0066C, &dmaHigh)); + + gcmkPRINT_N(0, "**************************\n"); + gcmkPRINT_N(0, "*** GPU STATE DUMP ***\n"); + gcmkPRINT_N(0, "**************************\n"); + + gcmkPRINT_N(4, " axi = 0x%08X\n", axi); + + gcmkPRINT_N(4, " idle = 0x%08X\n", idle); + if ((idle & 0x00000001) == 0) gcmkPRINT_N(0, " FE not idle\n"); + if ((idle & 0x00000002) == 0) gcmkPRINT_N(0, " DE not idle\n"); + if ((idle & 0x00000004) == 0) gcmkPRINT_N(0, " PE not idle\n"); + if ((idle & 0x00000008) == 0) gcmkPRINT_N(0, " SH not idle\n"); + if ((idle & 0x00000010) == 0) gcmkPRINT_N(0, " PA not idle\n"); + if ((idle & 0x00000020) == 0) gcmkPRINT_N(0, " SE not idle\n"); + if ((idle & 0x00000040) == 0) gcmkPRINT_N(0, " RA not idle\n"); + if ((idle & 0x00000080) == 0) gcmkPRINT_N(0, " TX not idle\n"); + if ((idle & 0x00000100) == 0) gcmkPRINT_N(0, " VG not idle\n"); + if ((idle & 0x00000200) == 0) gcmkPRINT_N(0, " IM not idle\n"); + if ((idle & 0x00000400) == 0) gcmkPRINT_N(0, " FP not idle\n"); + if ((idle & 0x00000800) == 0) gcmkPRINT_N(0, " TS not idle\n"); + if ((idle & 0x00001000) == 0) gcmkPRINT_N(0, " BL not idle\n"); + if ((idle & 0x00002000) == 0) gcmkPRINT_N(0, " ASYNCFE not idle\n"); + if ((idle & 0x00004000) == 0) gcmkPRINT_N(0, " MC not idle\n"); + if ((idle & 0x00008000) == 0) gcmkPRINT_N(0, " PPA not idle\n"); + if ((idle & 0x00010000) == 0) gcmkPRINT_N(0, " DC not idle\n"); + if ((idle & 0x00020000) == 0) gcmkPRINT_N(0, " WD not idle\n"); + if ((idle & 0x00040000) == 0) gcmkPRINT_N(0, " NN not idle\n"); + if ((idle & 0x00080000) == 0) gcmkPRINT_N(0, " TP not idle\n"); + if ((idle & 0x80000000) != 0) gcmkPRINT_N(0, " AXI low power mode\n"); + + if ((dmaAddress1 == dmaAddress2) + && (dmaState1 == dmaState2)) + { + gcmkPRINT_N(0, " DMA appears to be stuck at this address:\n"); + gcmkPRINT_N(4, " 0x%08X\n", dmaAddress1); + } + else + { + if (dmaAddress1 == dmaAddress2) + { + gcmkPRINT_N(0, " DMA address is constant, but state is changing:\n"); + gcmkPRINT_N(4, " 0x%08X\n", dmaState1); + gcmkPRINT_N(4, " 0x%08X\n", dmaState2); + } + else + { + gcmkPRINT_N(0, " DMA is running; known addresses are:\n"); + gcmkPRINT_N(4, " 0x%08X\n", dmaAddress1); + gcmkPRINT_N(4, " 0x%08X\n", dmaAddress2); + } + } + + gcmkPRINT_N(4, " dmaLow = 0x%08X\n", dmaLow); + gcmkPRINT_N(4, " dmaHigh = 0x%08X\n", dmaHigh); + gcmkPRINT_N(4, " dmaState = 0x%08X\n", dmaState2); + gcmkPRINT_N(8, " command state = %d (%s)\n", cmdState, _cmdState [cmdState]); + gcmkPRINT_N(8, " command DMA state = %d (%s)\n", cmdDmaState, _cmdDmaState[cmdDmaState]); + gcmkPRINT_N(8, " command fetch state = %d (%s)\n", cmdFetState, _cmdFetState[cmdFetState]); + gcmkPRINT_N(8, " DMA request state = %d (%s)\n", dmaReqState, _reqDmaState[dmaReqState]); + gcmkPRINT_N(8, " cal state = %d (%s)\n", calState, _calState [calState]); + gcmkPRINT_N(8, " VE request state = %d (%s)\n", veReqState, _veReqState [veReqState]); + + gcmkPRINT_N(0, " Debug registers:\n"); + + if (bltEngine) + { + _dbgRegs[BLT_INDEX].avail = gcvTRUE; + } + if (hwTFB) + { + _dbgRegs[TFB_INDEX].avail = gcvTRUE; + } + if (usc) + { + _dbgRegs[USC_INDEX].avail = gcvTRUE; + } + if (gsShader) + { + _dbgRegs[TPG_INDEX].avail = gcvTRUE; + } + if (multiCluster) + { + _dbgRegs[WD_INDEX].avail = gcvTRUE; + _dbgRegs[DIR_INDEX].avail = gcvTRUE; + _dbgRegs[VTXDATA_INDEX].avail = gcvTRUE; + _dbgRegs[PPA_INDEX].avail = gcvTRUE; + _dbgRegs[FE_INDEX].index = 0xF0; + _dbgRegs[HI_INDEX].index = 0xF0; + /*spare 64 DWORDS debug values from TX for VTXDATA prefetch in USC */ + _dbgRegs[TX_INDEX].count = 64; + + for (i = 0; i < gcmCOUNTOF(_dbgRegs); i++) + { + if (_dbgRegs[i].inCluster) + { + _dbgRegs[i].pipeMask = + Hardware->identity.clusterAvailMask & Hardware->options.userClusterMask; + } + } + pipeMask = Hardware->identity.clusterAvailMask & Hardware->options.userClusterMask; + } + if (nnEngine) + { + _dbgRegs[NN_INDEX].avail = gcvTRUE; + } + + for (i = 0; i < gcmCOUNTOF(_dbgRegs); i += 1) + { + gcmkONERROR(_DumpDebugRegisters(os, core, &_dbgRegs[i])); + } + + /* Record control. */ + gcmkONERROR(gckOS_ReadRegisterEx(os, core, 0x0, &oldControl)); + + for (pipe = 0; pipe < maxNumOfPipes; pipe++) + { + if (((1 << pipe) & pipeMask) == 0) + continue; + + gcmkPRINT_N(4, " Other Registers[%d]:\n", pipe); + + /* Switch pipe. */ + gcmkONERROR(gckOS_ReadRegisterEx(os, core, 0x0, &control)); + control &= ~(0xF << 20); + control |= (pipe << 20); + gcmkONERROR(gckOS_WriteRegisterEx(os, core, 0x0, control)); + + for (i = 0; i < gcmCOUNTOF(_otherRegs); i += 1) + { + gctUINT32 read; + gcmkONERROR(gckOS_ReadRegisterEx(os, core, _otherRegs[i], &read)); + gcmkPRINT_N(12, " [0x%04X] 0x%08X\n", _otherRegs[i], read); + } + + if (Hardware->mmuVersion) + { + gcmkPRINT(" MMU status from MC[%d]:", pipe); + gckHARDWARE_DumpMMUException(Hardware); + } + } + + /* MCFE state. */ + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_MCFE)) + { + gcmkVERIFY_OK(_DumpMCFEState(os, core)); + } + + /* Restore control. */ + gcmkONERROR(gckOS_WriteRegisterEx(os, core, 0x0, oldControl)); + + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_HALTI0)) + { + /* FE debug register. */ + gcmkVERIFY_OK(_DumpLinkStack(os, core, &_dbgRegs[2])); + } + + _DumpFEStack(os, core, &_dbgRegs[2]); + + gcmkPRINT_N(0, "**************************\n"); + gcmkPRINT_N(0, "***** SW COUNTERS *****\n"); + gcmkPRINT_N(0, "**************************\n"); + gcmkPRINT_N(4, " Execute Count = 0x%08X\n", Hardware->executeCount); + gcmkPRINT_N(4, " Execute Addr = 0x%08X\n", Hardware->lastExecuteAddress); + gcmkPRINT_N(4, " End Addr = 0x%08X\n", Hardware->lastEnd); + + /* dump stack. */ + gckOS_DumpCallStack(os); + +OnError: + + /* Return the error. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckHARDWARE_ReadPerformanceRegister( + IN gckHARDWARE Hardware, + IN gctUINT PerformanceAddress, + IN gctUINT IndexAddress, + IN gctUINT IndexShift, + IN gctUINT Index, + OUT gctUINT32_PTR Value + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x PerformanceAddress=0x%x IndexAddress=0x%x " + "IndexShift=%u Index=%u", + Hardware, PerformanceAddress, IndexAddress, IndexShift, + Index); + + /* Write the index. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + IndexAddress, + Index << IndexShift)); + + /* Read the register. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + PerformanceAddress, + Value)); + + /* Test for reset. */ + if (Index == 15) + { + /* Index another register to get out of reset. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, IndexAddress, 0)); + } + + /* Success. */ + gcmkFOOTER_ARG("*Value=0x%x", *Value); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_GetFrameInfo( + IN gckHARDWARE Hardware, + OUT gcsHAL_FRAME_INFO * FrameInfo + ) +{ + gceSTATUS status; + gctUINT i, clock; + gcsHAL_FRAME_INFO info; +#if gcdFRAME_DB_RESET + gctUINT reset; +#endif + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Get profile tick. */ + gcmkONERROR(gckOS_GetProfileTick(&info.ticks)); + + /* Read SH counters and reset them. */ + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0045C, + 0x00470, + 24, + 4, + &info.shaderCycles)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0045C, + 0x00470, + 24, + 9, + &info.vsInstructionCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0045C, + 0x00470, + 24, + 12, + &info.vsTextureCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0045C, + 0x00470, + 24, + 7, + &info.psInstructionCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0045C, + 0x00470, + 24, + 14, + &info.psTextureCount)); +#if gcdFRAME_DB_RESET + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0045C, + 0x00470, + 24, + 15, + &reset)); +#endif + + /* Read PA counters and reset them. */ + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00460, + 0x00474, + 0, + 3, + &info.vertexCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00460, + 0x00474, + 0, + 4, + &info.primitiveCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00460, + 0x00474, + 0, + 7, + &info.rejectedPrimitives)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00460, + 0x00474, + 0, + 8, + &info.culledPrimitives)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00460, + 0x00474, + 0, + 6, + &info.clippedPrimitives)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00460, + 0x00474, + 0, + 5, + &info.outPrimitives)); +#if gcdFRAME_DB_RESET + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00460, + 0x00474, + 0, + 15, + &reset)); +#endif + + /* Read RA counters and reset them. */ + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00448, + 0x00474, + 16, + 3, + &info.inPrimitives)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00448, + 0x00474, + 16, + 11, + &info.culledQuadCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00448, + 0x00474, + 16, + 1, + &info.totalQuadCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00448, + 0x00474, + 16, + 2, + &info.quadCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00448, + 0x00474, + 16, + 0, + &info.totalPixelCount)); +#if gcdFRAME_DB_RESET + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00448, + 0x00474, + 16, + 15, + &reset)); +#endif + + /* Read TX counters and reset them. */ + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0044C, + 0x00474, + 24, + 0, + &info.bilinearRequests)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0044C, + 0x00474, + 24, + 1, + &info.trilinearRequests)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0044C, + 0x00474, + 24, + 8, + &info.txHitCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0044C, + 0x00474, + 24, + 9, + &info.txMissCount)); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0044C, + 0x00474, + 24, + 6, + &info.txBytes8)); +#if gcdFRAME_DB_RESET + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x0044C, + 0x00474, + 24, + 15, + &reset)); +#endif + + /* Read clock control register. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + &clock)); + + /* Walk through all avaiable pixel pipes. */ + for (i = 0; i < Hardware->identity.pixelPipes; ++i) + { + /* Select proper pipe. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + ((((gctUINT32) (clock)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:20) - (0 ? + 23:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:20) - (0 ? + 23:20) + 1))))))) << (0 ? + 23:20))) | (((gctUINT32) ((gctUINT32) (i) & ((gctUINT32) ((((1 ? + 23:20) - (0 ? + 23:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:20) - (0 ? 23:20) + 1))))))) << (0 ? 23:20))))); + + /* Read cycle registers. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x0007C, + &info.idleCycles[i])); + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00078, + &info.cycles[i])); + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00438, + &info.mcCycles[i])); + + /* Read bandwidth registers. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x0005C, + &info.readRequests[i])); + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00040, + &info.readBytes8[i])); + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00050, + &info.writeRequests[i])); + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00044, + &info.writeBytes8[i])); + + /* Read PE counters. */ + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00454, + 0x00470, + 16, + 0, + &info.colorKilled[i])); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00454, + 0x00470, + 16, + 2, + &info.colorDrawn[i])); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00454, + 0x00470, + 16, + 1, + &info.depthKilled[i])); + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00454, + 0x00470, + 16, + 3, + &info.depthDrawn[i])); + } + + /* Zero out remaning reserved counters. */ + for (; i < 8; ++i) + { + info.readBytes8[i] = 0; + info.writeBytes8[i] = 0; + info.cycles[i] = 0; + info.idleCycles[i] = 0; + info.mcCycles[i] = 0; + info.readRequests[i] = 0; + info.writeRequests[i] = 0; + info.colorKilled[i] = 0; + info.colorDrawn[i] = 0; + info.depthKilled[i] = 0; + info.depthDrawn[i] = 0; + } + + /* Reset clock control register. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + clock)); + + /* Reset cycle and bandwidth counters. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0003C, + 1)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0003C, + 0)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00078, + 0)); + +#if gcdFRAME_DB_RESET + /* Reset PE counters. */ + gcmkONERROR(gckHARDWARE_ReadPerformanceRegister( + Hardware, + 0x00454, + 0x00470, + 16, + 15, + &reset)); +#endif + + /* Copy to user. */ + gcmkONERROR(gckOS_CopyToUserData(Hardware->os, + &info, + FrameInfo, + gcmSIZEOF(info))); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_DumpGpuProfile( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT clock, i; + gctUINT32 totalRead, totalWrite, read, write; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Read clock control register. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + &clock)); + + totalRead = 0; + totalWrite = 0; + + /* Walk through all avaiable pixel pipes. */ + for (i = 0; i < Hardware->identity.pixelPipes; ++i) + { + /* Select proper pipe. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + ((((gctUINT32) (clock)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:20) - (0 ? + 23:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:20) - (0 ? + 23:20) + 1))))))) << (0 ? + 23:20))) | (((gctUINT32) ((gctUINT32) (i) & ((gctUINT32) ((((1 ? + 23:20) - (0 ? + 23:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:20) - (0 ? 23:20) + 1))))))) << (0 ? 23:20))))); + + /* BW */ + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00040, + &read)); + totalRead += read; + + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00044, + &write)); + totalWrite += write; + } + + gcmkPRINT("==============GPU Profile: read request : %d\n", totalRead); + gcmkPRINT("==============GPU Profile: write request: %d\n", totalWrite); + + /* Reset clock control register. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00000, + clock)); + /* Reset counters. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x0003C, 1)); + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x0003C, 0)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +#if gcdDVFS +#define READ_FROM_EATER1 0 + +gceSTATUS +gckHARDWARE_QueryLoad( + IN gckHARDWARE Hardware, + OUT gctUINT32 * Load + ) +{ + gctUINT32 debug1; + gceSTATUS status; + gcmkHEADER_ARG("Hardware=0x%X", Hardware); + + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(Load != gcvNULL); + + gckOS_AcquireMutex(Hardware->os, Hardware->powerMutex, gcvINFINITE); + + if (Hardware->chipPowerState == gcvPOWER_ON) + { + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00110, + Load)); +#if READ_FROM_EATER1 + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00134, + Load)); +#endif + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00114, + &debug1)); + + /* Patch result of 0x110 with result of 0x114. */ + if ((debug1 & 0xFF) == 1) + { + *Load &= ~0xFF; + *Load |= 1; + } + + if (((debug1 & 0xFF00) >> 8) == 1) + { + *Load &= ~(0xFF << 8); + *Load |= 1 << 8; + } + + if (((debug1 & 0xFF0000) >> 16) == 1) + { + *Load &= ~(0xFF << 16); + *Load |= 1 << 16; + } + + if (((debug1 & 0xFF000000) >> 24) == 1) + { + *Load &= ~(0xFF << 24); + *Load |= 1 << 24; + } + } + else + { + status = gcvSTATUS_INVALID_REQUEST; + } + +OnError: + + gckOS_ReleaseMutex(Hardware->os, Hardware->powerMutex); + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_SetDVFSPeroid( + IN gckHARDWARE Hardware, + OUT gctUINT32 Frequency + ) +{ + gceSTATUS status; + gctUINT32 period; + gctUINT32 eater; + +#if READ_FROM_EATER1 + gctUINT32 period1; + gctUINT32 eater1; +#endif + + gcmkHEADER_ARG("Hardware=0x%X Frequency=%d", Hardware, Frequency); + + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + period = 0; + + while((64 << period) < (gcdDVFS_ANAYLSE_WINDOW * Frequency * 1000) ) + { + period++; + } + +#if READ_FROM_EATER1 + /* + * Peroid = F * 1000 * 1000 / (60 * 16 * 1024); + */ + period1 = Frequency * 6250 / 6114; +#endif + + gckOS_AcquireMutex(Hardware->os, Hardware->powerMutex, gcvINFINITE); + + if (Hardware->chipPowerState == gcvPOWER_ON) + { + /* Get current configure. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + &eater)); + + /* Change peroid. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + ((((gctUINT32) (eater)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:8) - (0 ? + 15:8) + 1))))))) << (0 ? + 15:8))) | (((gctUINT32) ((gctUINT32) (period) & ((gctUINT32) ((((1 ? + 15:8) - (0 ? + 15:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1))))))) << (0 ? 15:8))))); + +#if READ_FROM_EATER1 + /* Config eater1. */ + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x00130, + &eater1)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00130, + ((((gctUINT32) (eater1)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:16) - (0 ? + 31:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:16) - (0 ? + 31:16) + 1))))))) << (0 ? + 31:16))) | (((gctUINT32) ((gctUINT32) (period1) & ((gctUINT32) ((((1 ? + 31:16) - (0 ? + 31:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:16) - (0 ? 31:16) + 1))))))) << (0 ? 31:16))))); +#endif + } + else + { + status = gcvSTATUS_INVALID_REQUEST; + } + +OnError: + gckOS_ReleaseMutex(Hardware->os, Hardware->powerMutex); + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckHARDWARE_InitDVFS( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gctUINT32 data; + + gcmkHEADER_ARG("Hardware=0x%X", Hardware); + + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + &data)); + + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))); + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 18:18) - (0 ? + 18:18) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 18:18) - (0 ? + 18:18) + 1))))))) << (0 ? + 18:18))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 18:18) - (0 ? + 18:18) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 18:18) - (0 ? 18:18) + 1))))))) << (0 ? 18:18))); + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 19:19) - (0 ? + 19:19) + 1))))))) << (0 ? + 19:19))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 19:19) - (0 ? + 19:19) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 19:19) - (0 ? 19:19) + 1))))))) << (0 ? 19:19))); + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 20:20) - (0 ? + 20:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 20:20) - (0 ? + 20:20) + 1))))))) << (0 ? + 20:20))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 20:20) - (0 ? + 20:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 20:20) - (0 ? 20:20) + 1))))))) << (0 ? 20:20))); + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:23) - (0 ? + 23:23) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:23) - (0 ? + 23:23) + 1))))))) << (0 ? + 23:23))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 23:23) - (0 ? + 23:23) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:23) - (0 ? 23:23) + 1))))))) << (0 ? 23:23))); + data = ((((gctUINT32) (data)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 22:22) - (0 ? + 22:22) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 22:22) - (0 ? + 22:22) + 1))))))) << (0 ? + 22:22))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 22:22) - (0 ? + 22:22) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 22:22) - (0 ? 22:22) + 1))))))) << (0 ? 22:22))); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "DVFS Configure=0x%X", + data); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x0010C, + data)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} +#endif + +gceSTATUS +gckHARDWARE_ExecuteFunctions( + IN gcsFUNCTION_EXECUTION_PTR Execution + ) +{ + gceSTATUS status; + gctUINT32 idle; + gctUINT32 timer = 0, delay = 1; + gctUINT32 address; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + +#if gcdCAPTURE_ONLY_MODE + return gcvSTATUS_OK; +#endif + + address = Execution->address; + + /* Execute prepared command sequence. */ + if (hardware->mcFE) + { + gcmkONERROR(gckMCFE_Execute(hardware, gcvFALSE, 0, address, Execution->bytes)); + } + else + { + gcmkONERROR(gckWLFE_Execute(hardware, address, Execution->bytes)); + } + +#if gcdLINK_QUEUE_SIZE + { + gcuQUEUEDATA data; + + gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); + + data.linkData.start = address; + data.linkData.end = address + Execution->bytes; + data.linkData.linkLow = 0; + data.linkData.linkHigh = 0; + + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + } +#endif + + gckFUNCTION_Dump(Execution); + + /* Wait until GPU idle. */ + do + { + gckOS_Delay(hardware->os, delay); + + gcmkONERROR(gckOS_ReadRegisterEx( + hardware->os, + hardware->core, + 0x00004, + &idle)); + + timer += delay; + delay *= 2; + +#if gcdGPU_TIMEOUT + if (timer >= hardware->kernel->timeOut) + { + gckHARDWARE_DumpGPUState(hardware); + + if (hardware->kernel->command) + { + gckCOMMAND_DumpExecutingBuffer(hardware->kernel->command); + } + + /* Even if hardware is not reset correctly, let software + ** continue to avoid software stuck. Software will timeout again + ** and try to recover GPU in next timeout. + */ + gcmkONERROR(gcvSTATUS_DEVICE); + } +#endif + } + while (!_IsGPUIdle(idle)); + + return gcvSTATUS_OK; + +OnError: + return status; +} + +gceSTATUS +gckHARDWARE_QueryStateTimer( + IN gckHARDWARE Hardware, + OUT gctUINT64_PTR On, + OUT gctUINT64_PTR Off, + OUT gctUINT64_PTR Idle, + OUT gctUINT64_PTR Suspend + ) +{ + gckOS_AcquireMutex(Hardware->os, Hardware->powerMutex, gcvINFINITE); + + gckSTATETIMER_Query( + &Hardware->powerStateCounter, + Hardware->chipPowerState, + On, Off, Idle, Suspend); + + gckOS_ReleaseMutex(Hardware->os, Hardware->powerMutex); + + return gcvSTATUS_OK; +} + +gceSTATUS +gckHARDWARE_WaitFence( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT64 FenceData, + IN gctUINT32 FenceAddress, + OUT gctUINT32 *Bytes + ) +{ + gctUINT32_PTR logical = (gctUINT32_PTR)Logical; + + gctUINT32 dataLow = (gctUINT32)FenceData; + gctUINT32 dataHigh = (gctUINT32)(FenceData >> 32); + + if (logical) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x01FD) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = dataHigh; + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x01FA) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = dataLow; + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x0F & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (Hardware->waitCount) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:16) - (0 ? + 17:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:16) - (0 ? + 17:16) + 1))))))) << (0 ? + 17:16))) | (((gctUINT32) (0x2 & ((gctUINT32) ((((1 ? + 17:16) - (0 ? + 17:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:16) - (0 ? 17:16) + 1))))))) << (0 ? 17:16))); + + *logical++ + = FenceAddress; + } + else + { + *Bytes = 6 * gcmSIZEOF(gctUINT32); + } + + return gcvSTATUS_OK; +} + +gceSTATUS +gckHARDWARE_UpdateContextID( + IN gckHARDWARE Hardware + ) +{ + static gcsiDEBUG_REGISTERS fe = { "FE", 0x470, 0, 0x450, 256, 0x1, 0x00, gcvTRUE, gcvFALSE}; + gckOS os = Hardware->os; + gceCORE core = Hardware->core; + gctUINT32 contextIDLow, contextIDHigh; + gceSTATUS status; + + gcmkONERROR(gckOS_WriteRegisterEx(os, core, fe.index, 0x53 << fe.shift)); + gcmkONERROR(gckOS_ReadRegisterEx(os, core, fe.data, &contextIDLow)); + + gcmkONERROR(gckOS_WriteRegisterEx(os, core, fe.index, 0x54 << fe.shift)); + gcmkONERROR(gckOS_ReadRegisterEx(os, core, fe.data, &contextIDHigh)); + + Hardware->contextID = ((gctUINT64)contextIDHigh << 32) + contextIDLow; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +gceSTATUS +gckHARDWARE_DummyDraw( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN gceDUMMY_DRAW_TYPE DummyDrawType, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT32 dummyDraw_gc400[] = { + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0193) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0194) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0180) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:0) - (0 ? + 3:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:0) - (0 ? + 3:0) + 1))))))) << (0 ? + 3:0))) | (((gctUINT32) (0x8 & ((gctUINT32) ((((1 ? + 3:0) - (0 ? + 3:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:0) - (0 ? 3:0) + 1))))))) << (0 ? 3:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 13:12) - (0 ? + 13:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 13:12) - (0 ? + 13:12) + 1))))))) << (0 ? + 13:12))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 13:12) - (0 ? + 13:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 13:12) - (0 ? 13:12) + 1))))))) << (0 ? 13:12))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:24) - (0 ? + 31:24) + 1))))))) << (0 ? + 31:24))) | (((gctUINT32) ((gctUINT32) (4 * gcmSIZEOF(float)) & ((gctUINT32) ((((1 ? + 31:24) - (0 ? + 31:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:24) - (0 ? 31:24) + 1))))))) << (0 ? 31:24))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E05) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:0) - (0 ? + 1:0) + 1))))))) << (0 ? + 1:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:0) - (0 ? 1:0) + 1))))))) << (0 ? 1:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0202) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:0) - (0 ? + 5:0) + 1))))))) << (0 ? + 5:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:0) - (0 ? 5:0) + 1))))))) << (0 ? 5:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0208) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:0) - (0 ? + 5:0) + 1))))))) << (0 ? + 5:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:0) - (0 ? 5:0) + 1))))))) << (0 ? 5:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0201) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:0) - (0 ? + 5:0) + 1))))))) << (0 ? + 5:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:0) - (0 ? 5:0) + 1))))))) << (0 ? 5:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0204) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:0) - (0 ? + 5:0) + 1))))))) << (0 ? + 5:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 5:0) - (0 ? + 5:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:0) - (0 ? 5:0) + 1))))))) << (0 ? 5:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x1000) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x0, 0x0, 0x0, 0x0, + 0xDEADDEAD, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0203) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:0) - (0 ? + 6:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:0) - (0 ? + 6:0) + 1))))))) << (0 ? + 6:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 6:0) - (0 ? + 6:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:0) - (0 ? 6:0) + 1))))))) << (0 ? 6:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x020E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0200) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 1, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x020C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x000F003F, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x028C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 11:8) - (0 ? + 11:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 11:8) - (0 ? + 11:8) + 1))))))) << (0 ? + 11:8))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 11:8) - (0 ? + 11:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 11:8) - (0 ? 11:8) + 1))))))) << (0 ? 11:8))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0500) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:0) - (0 ? + 1:0) + 1))))))) << (0 ? + 1:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:0) - (0 ? 1:0) + 1))))))) << (0 ? 1:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x028D) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 13:12) - (0 ? + 13:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 13:12) - (0 ? + 13:12) + 1))))))) << (0 ? + 13:12))) | (((gctUINT32) (0x2 & ((gctUINT32) ((((1 ? + 13:12) - (0 ? + 13:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 13:12) - (0 ? 13:12) + 1))))))) << (0 ? 13:12))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:8) - (0 ? + 9:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:8) - (0 ? + 9:8) + 1))))))) << (0 ? + 9:8))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 9:8) - (0 ? + 9:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:8) - (0 ? 9:8) + 1))))))) << (0 ? 9:8))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:16) - (0 ? + 17:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:16) - (0 ? + 17:16) + 1))))))) << (0 ? + 17:16))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 17:16) - (0 ? + 17:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:16) - (0 ? 17:16) + 1))))))) << (0 ? 17:16))), + + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0300) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0301) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0302) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0303) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0289) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:0) - (0 ? + 3:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:0) - (0 ? + 3:0) + 1))))))) << (0 ? + 3:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 3:0) - (0 ? + 3:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:0) - (0 ? 3:0) + 1))))))) << (0 ? 3:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x05 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:0) - (0 ? + 3:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:0) - (0 ? + 3:0) + 1))))))) << (0 ? + 3:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 3:0) - (0 ? + 3:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:0) - (0 ? 3:0) + 1))))))) << (0 ? 3:0))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:0) - (0 ? + 23:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:0) - (0 ? + 23:0) + 1))))))) << (0 ? + 23:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 23:0) - (0 ? + 23:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:0) - (0 ? 23:0) + 1))))))) << (0 ? 23:0))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:0) - (0 ? + 23:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:0) - (0 ? + 23:0) + 1))))))) << (0 ? + 23:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 23:0) - (0 ? + 23:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:0) - (0 ? 23:0) + 1))))))) << (0 ? 23:0))), + }; + + gctUINT32 dummyDraw_v60[] = { + + /* Semaphore from FE to PE. */ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))), + + /* Stall from FE to PE. */ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x021A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (0x0) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E06) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:0) - (0 ? + 1:0) + 1))))))) << (0 ? + 1:0))) | (((gctUINT32) ((gctUINT32) (0x0) & ((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:0) - (0 ? 1:0) + 1))))))) << (0 ? 1:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 14:12) - (0 ? + 14:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 14:12) - (0 ? + 14:12) + 1))))))) << (0 ? + 14:12))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 14:12) - (0 ? + 14:12) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 14:12) - (0 ? 14:12) + 1))))))) << (0 ? 14:12))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 17:16) - (0 ? + 17:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 17:16) - (0 ? + 17:16) + 1))))))) << (0 ? + 17:16))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 17:16) - (0 ? + 17:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 17:16) - (0 ? 17:16) + 1))))))) << (0 ? 17:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:4) - (0 ? + 7:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:4) - (0 ? + 7:4) + 1))))))) << (0 ? + 7:4))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 7:4) - (0 ? + 7:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:4) - (0 ? 7:4) + 1))))))) << (0 ? 7:4))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0401) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (6) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x0, + 0x2, + 0x0, + 0x0, + 0x0, + 0x0, + (gctUINT32)~0x0, + + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x020C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0xffffffff, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E07) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 2, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E08) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 2, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0420) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:0) - (0 ? + 2:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:0) - (0 ? + 2:0) + 1))))))) << (0 ? + 2:0))) | (((gctUINT32) ((gctUINT32) (2) & ((gctUINT32) ((((1 ? + 2:0) - (0 ? + 2:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:0) - (0 ? 2:0) + 1))))))) << (0 ? 2:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0424) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 1, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0403) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 3, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E21) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:0) - (0 ? + 2:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:0) - (0 ? + 2:0) + 1))))))) << (0 ? + 2:0))) | (((gctUINT32) (0x2 & ((gctUINT32) ((((1 ? + 2:0) - (0 ? + 2:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:0) - (0 ? 2:0) + 1))))))) << (0 ? 2:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x040A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x2000) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1 << 2) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x07801033,0x3fc00900,0x00000040,0x00390008, + (gctUINT32)~0, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x021F) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x0, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0240) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:0) - (0 ? + 1:0) + 1))))))) << (0 ? + 1:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 1:0) - (0 ? + 1:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:0) - (0 ? 1:0) + 1))))))) << (0 ? 1:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:4) - (0 ? + 6:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:4) - (0 ? + 6:4) + 1))))))) << (0 ? + 6:4))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 6:4) - (0 ? + 6:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:4) - (0 ? 6:4) + 1))))))) << (0 ? 6:4))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 8:8) - (0 ? + 8:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 8:8) - (0 ? + 8:8) + 1))))))) << (0 ? + 8:8))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 8:8) - (0 ? + 8:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 8:8) - (0 ? 8:8) + 1))))))) << (0 ? 8:8))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 26:24) - (0 ? + 26:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 26:24) - (0 ? + 26:24) + 1))))))) << (0 ? + 26:24))) | (((gctUINT32) (0x3 & ((gctUINT32) ((((1 ? + 26:24) - (0 ? + 26:24) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 26:24) - (0 ? 26:24) + 1))))))) << (0 ? 26:24))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0241) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (31) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:16) - (0 ? + 31:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:16) - (0 ? + 31:16) + 1))))))) << (0 ? + 31:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 31:16) - (0 ? + 31:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:16) - (0 ? 31:16) + 1))))))) << (0 ? 31:16))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0244) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 9:0) - (0 ? + 9:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 9:0) - (0 ? + 9:0) + 1))))))) << (0 ? + 9:0))) | (((gctUINT32) ((gctUINT32) (31) & ((gctUINT32) ((((1 ? + 9:0) - (0 ? + 9:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 9:0) - (0 ? 9:0) + 1))))))) << (0 ? 9:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:16) - (0 ? + 31:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:16) - (0 ? + 31:16) + 1))))))) << (0 ? + 31:16))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 31:16) - (0 ? + 31:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:16) - (0 ? 31:16) + 1))))))) << (0 ? 31:16))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0247) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + + (32+(4*(((gcsFEATURE_DATABASE *)Hardware->featureDatabase)->NumShaderCores)-1))/(4*(((gcsFEATURE_DATABASE *)Hardware->featureDatabase)->NumShaderCores)), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0248) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 1, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))), + + /* Semaphore from FE to PE. */ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))), + + /* Stall from FE to PE. */ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))), + + /* Invalidate I cache.*/ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x022C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 1:1) - (0 ? + 1:1) + 1))))))) << (0 ? + 1:1))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 1:1) - (0 ? + 1:1) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 1:1) - (0 ? 1:1) + 1))))))) << (0 ? 1:1))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 2:2) - (0 ? + 2:2) + 1))))))) << (0 ? + 2:2))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 2:2) - (0 ? + 2:2) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 2:2) - (0 ? 2:2) + 1))))))) << (0 ? 2:2))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 3:3) - (0 ? + 3:3) + 1))))))) << (0 ? + 3:3))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 3:3) - (0 ? + 3:3) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 3:3) - (0 ? 3:3) + 1))))))) << (0 ? 3:3))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:4) - (0 ? + 4:4) + 1))))))) << (0 ? + 4:4))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 4:4) - (0 ? + 4:4) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:4) - (0 ? 4:4) + 1))))))) << (0 ? 4:4))), + + /* SubmitJob. */ + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + }; + + gctUINT32 bytes = 0; + gctUINT32_PTR dummyDraw = gcvNULL; + + + switch(DummyDrawType) + { + case gcvDUMMY_DRAW_GC400: + dummyDraw = dummyDraw_gc400; + bytes = gcmSIZEOF(dummyDraw_gc400); + *(dummyDraw + 1) = Address; + break; + case gcvDUMMY_DRAW_V60: + dummyDraw = dummyDraw_v60; + bytes = gcmSIZEOF(dummyDraw_v60); + if (_QueryFeatureDatabase(Hardware, gcvFEATURE_MCFE)) + { + gctUINT32 submitJob; + + submitJob = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x16 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) (0x001 & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + if (bytes & 8) + { + /* To keep 16 byte alignment. */ + bytes -= 8; + } + + dummyDraw[(bytes >> 2) - 2] = submitJob; + } + break; + default: + /* other chip no need dummy draw.*/ + gcmkASSERT(0); + break; + }; + + if (Logical != gcvNULL) + { + gckOS_MemCopy(Logical, dummyDraw, bytes); + } + + *Bytes = bytes; + + return gcvSTATUS_OK; +} + +gceSTATUS +gckHARDWARE_EnterQueryClock( + IN gckHARDWARE Hardware, + OUT gctUINT64 *McStart, + OUT gctUINT64 *ShStart + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT64 mcStart, shStart; + + gcmkONERROR(gckOS_GetTime(&mcStart)); + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00438, 0)); + + *McStart = mcStart; + + if (Hardware->core <= gcvCORE_3D_MAX) + { + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, 0xFFU << 24)); + + gcmkONERROR(gckOS_GetTime(&shStart)); + + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00470, 0x4U << 24)); + + *ShStart = shStart; + } + +OnError: + return status; +} + +gceSTATUS +gckHARDWARE_ExitQueryClock( + IN gckHARDWARE Hardware, + IN gctUINT64 McStart, + IN gctUINT64 ShStart, + OUT gctUINT32 *McClk, + OUT gctUINT32 *ShClk + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT64 mcEnd, shEnd; + gctUINT32 mcCycle, shCycle; + gctUINT64 mcFreq, shFreq = 0; + + gcmkONERROR(gckOS_GetTime(&mcEnd)); + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00438, &mcCycle)); + + if (mcCycle == 0) + { + gcmkONERROR(gcvSTATUS_GENERIC_IO); + } + + /* cycle = (gctUINT64)cycle * 1000000 / (end - start); */ + mcFreq = ((gctUINT64)mcCycle * ((1000000U << 12) / (gctUINT32)(mcEnd - McStart))) >> 12; + + *McClk = (gctUINT32)mcFreq; + + if (Hardware->core <= gcvCORE_3D_MAX) + { + gcmkONERROR(gckOS_GetTime(&shEnd)); + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x0045C, &shCycle)); + + if (!shCycle) + { + /*TODO: [VIV] Query SH cycle not support for old chips */ + *ShClk = *McClk; + return gcvSTATUS_OK; + } + + if (!ShStart) + { + gcmkONERROR(gcvSTATUS_GENERIC_IO); + } + + shFreq = ((gctUINT64)shCycle * ((1000000U << 12) / (gctUINT32)(shEnd - ShStart))) >> 12; + } + + *ShClk = (gctUINT32)shFreq; + +OnError: + return status; +} + +/******************************************************************************* +** +** gckHARDWARE_QueryFrequency +** +** Query current hardware frequency. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +*/ +gceSTATUS +gckHARDWARE_QueryFrequency( + IN gckHARDWARE Hardware + ) +{ + gctUINT64 mcStart, shStart; + gctUINT32 mcClk, shClk; + gceSTATUS status; + gctUINT64 powerManagement = 0; + gctBOOL globalAcquired = gcvFALSE; + gceCHIPPOWERSTATE statesStored, state; + + gcmkHEADER_ARG("Hardware=0x%p", Hardware); + + gcmkVERIFY_ARGUMENT(Hardware != gcvNULL); + + mcStart = shStart = 0; + mcClk = shClk = 0; + + status = gckOS_QueryOption(Hardware->os, "powerManagement", &powerManagement); + if (gcmIS_ERROR(status)) + { + powerManagement = 0; + } + + if (powerManagement) + { + gcmkONERROR(gckHARDWARE_EnablePowerManagement( + Hardware, gcvFALSE + )); + } + + gcmkONERROR(gckHARDWARE_QueryPowerState( + Hardware, &statesStored + )); + + gcmkONERROR(gckHARDWARE_SetPowerState( + Hardware, gcvPOWER_ON_AUTO + )); + + /* Grab the global semaphore. */ + gcmkONERROR(gckOS_AcquireSemaphore( + Hardware->os, Hardware->globalSemaphore + )); + + globalAcquired = gcvTRUE; + + gckHARDWARE_EnterQueryClock(Hardware, &mcStart, &shStart); + + gcmkONERROR(gckOS_Delay(Hardware->os, 50)); + + if (mcStart) + { + gckHARDWARE_ExitQueryClock(Hardware, + mcStart, shStart, + &mcClk, &shClk); + + Hardware->mcClk = mcClk; + Hardware->shClk = shClk; + } + + /* Release the global semaphore. */ + gcmkONERROR(gckOS_ReleaseSemaphore( + Hardware->os, Hardware->globalSemaphore + )); + + globalAcquired = gcvFALSE; + + switch(statesStored) + { + case gcvPOWER_OFF: + state = gcvPOWER_OFF_BROADCAST; + break; + case gcvPOWER_IDLE: + state = gcvPOWER_IDLE_BROADCAST; + break; + case gcvPOWER_SUSPEND: + state = gcvPOWER_SUSPEND_BROADCAST; + break; + case gcvPOWER_ON: + state = gcvPOWER_ON_AUTO; + break; + default: + state = statesStored; + break; + } + + if (powerManagement) + { + gcmkONERROR(gckHARDWARE_EnablePowerManagement( + Hardware, gcvTRUE + )); + } + + gcmkONERROR(gckHARDWARE_SetPowerState( + Hardware, state + )); + + gcmkFOOTER_NO(); + + return gcvSTATUS_OK; + +OnError: + if (globalAcquired) + { + /* Release the global semaphore. */ + gcmkVERIFY_OK(gckOS_ReleaseSemaphore( + Hardware->os, Hardware->globalSemaphore + )); + } + + gcmkFOOTER(); + + return status; +} + +
diff --git a/hal/kernel/arch/gc_hal_kernel_hardware.h b/hal/kernel/arch/gc_hal_kernel_hardware.h new file mode 100644 index 0000000..83dfd6d --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_hardware.h
@@ -0,0 +1,313 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_kernel_hardware_h_ +#define __gc_hal_kernel_hardware_h_ + +#include "gc_hal_kernel_hardware_func.h" + + +#ifdef __cplusplus +extern "C" { +#endif + +#define EVENT_ID_INVALIDATE_PIPE 29 + +typedef enum { + gcvHARDWARE_FUNCTION_MMU, + gcvHARDWARE_FUNCTION_FLUSH, + + gcvHARDWARE_FUNCTION_NUM, +} +gceHARDWARE_FUNCTION; + +typedef struct _gckASYNC_FE * gckASYNC_FE; +typedef struct _gckWLFE * gckWLFE; +typedef struct _gckMCFE * gckMCFE; + +typedef struct _gcsSTATETIMER +{ + gctUINT64 start; + gctUINT64 recent; + + /* Elapse of each power state. */ + gctUINT64 elapse[4]; +} +gcsSTATETIMER; + +typedef struct _gcsHARDWARE_SIGNATURE +{ + /* Chip model. */ + gceCHIPMODEL chipModel; + + /* Revision value.*/ + gctUINT32 chipRevision; + + /* Supported feature fields. */ + gctUINT32 chipFeatures; + + /* Supported minor feature fields. */ + gctUINT32 chipMinorFeatures; + + /* Supported minor feature 1 fields. */ + gctUINT32 chipMinorFeatures1; + + /* Supported minor feature 2 fields. */ + gctUINT32 chipMinorFeatures2; +} +gcsHARDWARE_SIGNATURE; + +typedef struct _gcsMMU_TABLE_ARRAY_ENTRY +{ + gctUINT32 low; + gctUINT32 high; +} +gcsMMU_TABLE_ARRAY_ENTRY; + +typedef struct _gcsHARDWARE_PAGETABLE_ARRAY +{ + /* Number of entries in page table array. */ + gctUINT num; + + /* Video memory node. */ + gckVIDMEM_NODE videoMem; + + /* Size in bytes of array. */ + gctSIZE_T size; + + /* Physical address of array. */ + gctPHYS_ADDR_T address; + + /* Logical address of array. */ + gctPOINTER logical; +} +gcsHARDWARE_PAGETABLE_ARRAY; + +/* gckHARDWARE object. */ +struct _gckHARDWARE +{ + /* Object. */ + gcsOBJECT object; + + /* Pointer to gctKERNEL object. */ + gckKERNEL kernel; + + /* Pointer to gctOS object. */ + gckOS os; + + /* Core */ + gceCORE core; + + /* Type */ + gceHARDWARE_TYPE type; + + /* Chip characteristics. */ + gcsHAL_QUERY_CHIP_IDENTITY identity; + gcsHAL_QUERY_CHIP_OPTIONS options; + gctUINT32 powerBaseAddress; + gctBOOL extraEventStates; + + /* Big endian */ + gctBOOL bigEndian; + + /* Base address. */ + gctUINT32 baseAddress; + + /* FE modules. */ + gckWLFE wlFE; + gckASYNC_FE asyncFE; + gckMCFE mcFE; + + /* Chip status */ + gctPOINTER powerMutex; + gceCHIPPOWERSTATE chipPowerState; + gctBOOL clockState; + gctBOOL powerState; + gctPOINTER globalSemaphore; + gctBOOL isLastPowerGlobal; + + /* Wait Link FE only. */ + gctUINT32 lastWaitLink; + gctUINT32 lastEnd; + + gctUINT32 mmuVersion; + + gceCHIPPOWERSTATE nextPowerState; + gctPOINTER powerStateTimer; + +#if gcdENABLE_FSCALE_VAL_ADJUST + gctUINT32 powerOnFscaleVal; +#endif + gctPOINTER pageTableDirty[gcvENGINE_GPU_ENGINE_COUNT]; + +#if gcdLINK_QUEUE_SIZE + struct _gckQUEUE linkQueue; +#endif + gctBOOL stallFEPrefetch; + + gctUINT32 minFscaleValue; + gctUINT waitCount; + + gctUINT32 mcClk; + gctUINT32 shClk; + + gctPOINTER pendingEvent; + + gcsFUNCTION_EXECUTION_PTR functions; + + gcsSTATETIMER powerStateCounter; + gctUINT32 executeCount; + gctUINT32 lastExecuteAddress; + + /* Head for hardware list in gckMMU. */ + gcsLISTHEAD mmuHead; + + /* Internal SRAMs info. */ + gckVIDMEM sRAMVidMem[gcvSRAM_INTER_COUNT]; + gctPHYS_ADDR sRAMPhysical[gcvSRAM_INTER_COUNT]; + + gctPOINTER featureDatabase; + gctBOOL hasL2Cache; + + /* MCFE channel bindings, temporary. */ + gceMCFE_CHANNEL_TYPE mcfeChannels[64]; + gctUINT32 mcfeChannelCount; + + gcsHARDWARE_SIGNATURE signature; + + gctUINT32 maxOutstandingReads; + + gcsHARDWARE_PAGETABLE_ARRAY pagetableArray; + + gctUINT64 contextID; +}; + +gceSTATUS +gckHARDWARE_GetBaseAddress( + IN gckHARDWARE Hardware, + OUT gctUINT32_PTR BaseAddress + ); + +gceSTATUS +gckHARDWARE_NeedBaseAddress( + IN gckHARDWARE Hardware, + IN gctUINT32 State, + OUT gctBOOL_PTR NeedBase + ); + +gceSTATUS +gckHARDWARE_GetFrameInfo( + IN gckHARDWARE Hardware, + OUT gcsHAL_FRAME_INFO * FrameInfo + ); + +gceSTATUS +gckHARDWARE_DumpGpuProfile( + IN gckHARDWARE Hardware + ); + +gceSTATUS +gckHARDWARE_HandleFault( + IN gckHARDWARE Hardware + ); + +gceSTATUS +gckHARDWARE_ExecuteFunctions( + IN gcsFUNCTION_EXECUTION_PTR Execution + ); + +gceSTATUS +gckHARDWARE_DummyDraw( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN gceDUMMY_DRAW_TYPE DummyDrawType, + IN OUT gctUINT32 * Bytes + ); + +gceSTATUS +gckHARDWARE_EnterQueryClock( + IN gckHARDWARE Hardware, + OUT gctUINT64 *McStart, + OUT gctUINT64 *ShStart + ); + +gceSTATUS +gckHARDWARE_ExitQueryClock( + IN gckHARDWARE Hardware, + IN gctUINT64 McStart, + IN gctUINT64 ShStart, + OUT gctUINT32 *McClk, + OUT gctUINT32 *ShClk + ); + +gceSTATUS +gckHARDWARE_QueryFrequency( + IN gckHARDWARE Hardware + ); + +#define gcmkWRITE_MEMORY(logical, data) \ + do { \ + gcmkVERIFY_OK(gckOS_WriteMemory(os, logical, data)); \ + logical++; \ + }\ + while (0) ; \ + +#ifdef __cplusplus +} +#endif + +#endif /* __gc_hal_kernel_hardware_h_ */ +
diff --git a/hal/kernel/arch/gc_hal_kernel_hardware_async_fe.c b/hal/kernel/arch/gc_hal_kernel_hardware_async_fe.c new file mode 100644 index 0000000..a339e5a --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_hardware_async_fe.c
@@ -0,0 +1,670 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal.h" +#include "gc_hal_kernel.h" +#include "gc_hal_kernel_context.h" + +#define _GC_OBJ_ZONE gcvZONE_HARDWARE + +struct _gckASYNC_FE +{ + /* Number of free descriptors. */ + gctPOINTER freeDscriptors; +}; + +gceSTATUS +gckASYNC_FE_Construct( + IN gckHARDWARE Hardware, + OUT gckASYNC_FE * FE + ) +{ + gceSTATUS status; + gctUINT32 data; + gckASYNC_FE fe; + + gcmkHEADER(); + + gcmkONERROR(gckOS_Allocate(Hardware->os, + gcmSIZEOF(struct _gckASYNC_FE), + (gctPOINTER *)&fe)); + gckOS_ZeroMemory(fe, gcmSIZEOF(struct _gckASYNC_FE)); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x007E4, + &data + )); + + gcmkONERROR(gckOS_AtomConstruct(Hardware->os, &fe->freeDscriptors)); + + data = (((((gctUINT32) (data)) >> (0 ? 6:0)) & ((gctUINT32) ((((1 ? 6:0) - (0 ? 6:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 6:0) - (0 ? 6:0) + 1)))))) ); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, _GC_OBJ_ZONE, "free descriptor=%d", data); + + gcmkONERROR(gckOS_AtomSet(Hardware->os, fe->freeDscriptors, data)); + + /* Enable interrupts. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x000D8, ~0U)); + + *FE = fe; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (fe) + { + if (fe->freeDscriptors) + { + gckOS_AtomDestroy(Hardware->os, fe->freeDscriptors); + } + gcmkOS_SAFE_FREE(Hardware->os, fe); + } + + gcmkFOOTER(); + return status; +} + +void +gckASYNC_FE_Destroy( + IN gckHARDWARE Hardware, + IN gckASYNC_FE FE + ) +{ + if (FE->freeDscriptors) + { + gcmkOS_SAFE_FREE(Hardware->os, FE->freeDscriptors); + } + + gcmkOS_SAFE_FREE(Hardware->os, FE); +} + +gceSTATUS +gckASYNC_FE_Initialize( + IN gckHARDWARE Hardware, + IN gckASYNC_FE FE + ) +{ + return gcvSTATUS_OK; +} + +gceSTATUS +gckASYNC_FE_Nop( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctSIZE_T * Bytes + ) +{ + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x *Bytes=%lu", + Hardware, Logical, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + + if (Logical != gcvNULL) + { + if (*Bytes < 8) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + /* Append NOP. */ + logical[0] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, "0x%x: NOP", Logical); + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the NOP command. */ + *Bytes = 8; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckASYNC_FE_Event( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT8 Event, + IN gceKERNEL_WHERE FromWhere, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT size; + gctUINT32 destination = 0; + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + gctBOOL blt; + gctBOOL extraEventStates; + gctBOOL multiCluster; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x Event=%u FromWhere=%d *Bytes=%lu", + Hardware, Logical, Event, FromWhere, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + gcmkVERIFY_ARGUMENT(Event < 32); + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_BLT_ENGINE)) + { + /* Send all event from blt. */ + if (FromWhere == gcvKERNEL_PIXEL) + { + FromWhere = gcvKERNEL_BLT; + } + } + + blt = FromWhere == gcvKERNEL_BLT ? gcvTRUE : gcvFALSE; + + multiCluster = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_MULTI_CLUSTER); + + /* Determine the size of the command. */ + + extraEventStates = Hardware->extraEventStates && (FromWhere == gcvKERNEL_PIXEL); + + size = extraEventStates + ? gcmALIGN(8 + (1 + 5) * 4, 8) /* EVENT + 5 STATES */ + : 8; + + if (blt) + { + size += 16; + if (multiCluster) + size += 8; + } + + if (Logical != gcvNULL) + { + if (*Bytes < size) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + switch (FromWhere) + { + case gcvKERNEL_COMMAND: + /* From command processor. */ + destination = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))); + break; + + case gcvKERNEL_PIXEL: + /* From pixel engine. */ + destination = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:6) - (0 ? + 6:6) + 1))))))) << (0 ? + 6:6))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:6) - (0 ? 6:6) + 1))))))) << (0 ? 6:6))); + break; + + case gcvKERNEL_BLT: + destination = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7))); + break; + + default: + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + if (blt) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + if (multiCluster) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x50CE) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (Hardware->identity.clusterAvailMask & Hardware->options.userClusterMask) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))); + } + } + + /* Append EVENT(Event, destination). */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E01) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = ((((gctUINT32) (destination)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) ((gctUINT32) (Event) & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))); + + if (blt) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + + + /* Make sure the event ID gets written out before GPU can access it. */ + gcmkONERROR( + gckOS_MemoryBarrier(Hardware->os, logical + 1)); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + { + gctPHYS_ADDR_T phys; + gckOS_GetPhysicalAddress(Hardware->os, Logical, &phys); + gckOS_CPUPhysicalToGPUPhysical(Hardware->os, phys, &phys); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%08x: EVENT %d", phys, Event); + } +#endif + + /* Append the extra states. These are needed for the chips that do not + ** support back-to-back events due to the async interface. The extra + ** states add the necessary delay to ensure that event IDs do not + ** collide. */ + if (extraEventStates) + { + *logical++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0100) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (5) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + *logical++ = 0; + *logical++ = 0; + *logical++ = 0; + *logical++ = 0; + *logical++ = 0; + } + +#if gcdINTERRUPT_STATISTIC + if (Event < gcmCOUNTOF(Hardware->kernel->eventObj->queues)) + { + gckOS_AtomSetMask(Hardware->pendingEvent, 1 << Event); + } +#endif + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the EVENT command. */ + *Bytes = size; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + + +void +gckASYNC_FE_UpdateAvaiable( + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status; + gctUINT32 data; + gctINT32 oldValue; + gckASYNC_FE fe = Hardware->asyncFE; + + status = gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x007E4, + &data + ); + + if (gcmIS_SUCCESS(status)) + { + data = (((((gctUINT32) (data)) >> (0 ? 6:0)) & ((gctUINT32) ((((1 ? 6:0) - (0 ? 6:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 6:0) - (0 ? 6:0) + 1)))))) ); + + while (data--) + { + gckOS_AtomIncrement(Hardware->os, fe->freeDscriptors, &oldValue); + } + } +} + +gceSTATUS +gckASYNC_FE_ReserveSlot( + IN gckHARDWARE Hardware, + OUT gctBOOL * Available + ) +{ + gctINT32 oldValue; + gckASYNC_FE fe = Hardware->asyncFE; + + gckOS_AtomDecrement(Hardware->os, fe->freeDscriptors, &oldValue); + + if (oldValue > 0) + { + /* Get one slot. */ + *Available = gcvTRUE; + } + else + { + /* No available slot, restore decreased one.*/ + gckOS_AtomIncrement(Hardware->os, fe->freeDscriptors, &oldValue); + *Available = gcvFALSE; + } + + return gcvSTATUS_OK; +} + +gceSTATUS +gckASYNC_FE_Execute( + IN gckHARDWARE Hardware, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ) +{ + gceSTATUS status; + + status = gckOS_WriteRegisterEx( + Hardware->os, + Hardware->core, + 0x007DC, + Address + ); + if (gcmIS_ERROR(status)) + { + return status; + } + + gckOS_MemoryBarrier( + Hardware->os, + gcvNULL + ); + + status = gckOS_WriteRegisterEx( + Hardware->os, + Hardware->core, + 0x007E0, + Address + Bytes + ); + if (gcmIS_ERROR(status)) + { + return status; + } + + return gcvSTATUS_OK; +} +
diff --git a/hal/kernel/arch/gc_hal_kernel_hardware_fe.h b/hal/kernel/arch/gc_hal_kernel_hardware_fe.h new file mode 100644 index 0000000..cc397e8 --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_hardware_fe.h
@@ -0,0 +1,296 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_kernel_hardware_fe_h_ +#define __gc_hal_kernel_hardware_fe_h_ + +#include "gc_hal.h" + +/******************************************************************************/ +/* Wait-Link FE commands. */ + +/* Construct Wait-Link FE. */ +gceSTATUS +gckWLFE_Construct( + IN gckHARDWARE Hardware, + OUT gckWLFE * FE + ); + +void +gckWLFE_Destroy( + IN gckHARDWARE Hardware, + IN gckWLFE FE + ); + +/* Initialize Wait-Link FE, when hardware reset. */ +gceSTATUS +gckWLFE_Initialize( + IN gckHARDWARE Hardware, + IN gckWLFE FE + ); + + +/* Add a WAIT/LINK pair in the command queue. */ +gceSTATUS +gckWLFE_WaitLink( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN gctUINT32 Offset, + IN OUT gctUINT32 * Bytes, + OUT gctUINT32 * WaitOffset, + OUT gctUINT32 * WaitBytes + ); + +/* Add a LINK command in the command queue. */ +gceSTATUS +gckWLFE_Link( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 FetchAddress, + IN gctUINT32 FetchSize, + IN OUT gctUINT32 * Bytes, + OUT gctUINT32 * Low, + OUT gctUINT32 * High + ); + +/* Add an END command in the command queue. */ +gceSTATUS +gckWLFE_End( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN OUT gctUINT32 * Bytes + ); + +/* Add a NOP command in the command queue. */ +gceSTATUS +gckWLFE_Nop( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctSIZE_T * Bytes + ); + +/* Add an EVENT command in the command queue. */ +gceSTATUS +gckWLFE_Event( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT8 Event, + IN gceKERNEL_WHERE FromWhere, + IN OUT gctUINT32 * Bytes + ); + +gceSTATUS +gckWLFE_ChipEnable( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gceCORE_3D_MASK ChipEnable, + IN OUT gctSIZE_T * Bytes + ); + +/* Kickstart the command processor. */ +gceSTATUS +gckWLFE_Execute( + IN gckHARDWARE Hardware, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ); + +/* Atomic version or IRQ routine. */ +gceSTATUS +gckWLFE_AtomicExecute( + IN gckHARDWARE Hardware, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ); + +/******************************************************************************/ +/* ASync FE commands. */ + +gceSTATUS +gckASYNC_FE_Construct( + IN gckHARDWARE Hardware, + OUT gckASYNC_FE * FE + ); + +void +gckASYNC_FE_Destroy( + IN gckHARDWARE Hardware, + IN gckASYNC_FE FE + ); + +/* Initialize Async FE, when hardware reset. */ +gceSTATUS +gckASYNC_FE_Initialize( + IN gckHARDWARE Hardware, + IN gckASYNC_FE FE + ); + +/* Add a NOP command in the command queue. */ +gceSTATUS +gckASYNC_FE_Nop( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctSIZE_T * Bytes + ); + +/* Add an EVENT command in the command queue. */ +gceSTATUS +gckASYNC_FE_Event( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT8 Event, + IN gceKERNEL_WHERE FromWhere, + IN OUT gctUINT32 * Bytes + ); + +/* Kickstart the command processor. */ +gceSTATUS +gckASYNC_FE_Execute( + IN gckHARDWARE Hardware, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ); + +gceSTATUS +gckASYNC_FE_ReserveSlot( + IN gckHARDWARE Hardware, + OUT gctBOOL * Available + ); + +void +gckASYNC_FE_UpdateAvaiable( + IN gckHARDWARE Hardware + ); + +/******************************************************************************/ +/* MC FE commands. */ + +/* One MCFE includes max 64 engine, each engine contains 2 channels. */ +gceSTATUS +gckMCFE_Construct( + IN gckHARDWARE Hardware, + OUT gckMCFE * FE + ); + +void +gckMCFE_Destroy( + IN gckHARDWARE Hardware, + IN gckMCFE FE + ); + +/* Initialize MC FE, when hardware reset. */ +gceSTATUS +gckMCFE_Initialize( + IN gckHARDWARE Hardware, + IN gctBOOL MMUEnabled, + IN gckMCFE FE + ); + +/* Add a NOP command in the command queue. */ +gceSTATUS +gckMCFE_Nop( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctSIZE_T * Bytes + ); + +/* Add an EVENT command in the command queue. */ +gceSTATUS +gckMCFE_Event( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT8 Event, + IN gceKERNEL_WHERE FromWhere, + IN OUT gctUINT32 * Bytes + ); + +/* Add a SendSemaphore command in the command queue. */ +gceSTATUS +gckMCFE_SendSemaphore( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 SemaId, + IN OUT gctUINT32 * Bytes + ); + +/* Add a WaitSemaphore command in the command queue. */ +gceSTATUS +gckMCFE_WaitSemaphore( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 SemaId, + IN OUT gctUINT32 * Bytes + ); + +/* Kickstart the command processor. */ +gceSTATUS +gckMCFE_Execute( + IN gckHARDWARE Hardware, + IN gctBOOL Priority, + IN gctUINT32 ChannelId, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ); + +/* Query hardware module idle */ +gceSTATUS +gckMCFE_HardwareIdle( + IN gckHARDWARE Hardware, + OUT gctBOOL_PTR IsIdle + ); +#endif +
diff --git a/hal/kernel/arch/gc_hal_kernel_hardware_func.c b/hal/kernel/arch/gc_hal_kernel_hardware_func.c new file mode 100644 index 0000000..61ca8d8 --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_hardware_func.c
@@ -0,0 +1,15161 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal.h" +#include "gc_hal_kernel.h" +#include "gc_hal_kernel_hardware.h" + + +#if gcdINITIALIZE_PPU +static gctUINT32 ppuMem0[] = +{ + 0x00010000, +}; + +static gctUINT32 ppuMem1[] = +{ + 0x00010000, +}; + +static gctUINT32 ppuMem2[] = +{ + 0x38011039, + 0x39200804, + 0xC0A90050, + 0x74000008, + 0x38021039, + 0x39202804, + 0xC0A90050, + 0x74000008, + 0x38031039, + 0x39204804, + 0xC0A90050, + 0x74000008, + 0xB8017005, + 0x002018FC, + 0x01C900C0, + 0x40390098, + 0xB8027005, + 0x002028FC, + 0x01C90140, + 0x40390098, + 0xB8037005, + 0x002038FC, + 0x01C901C0, + 0x40390098, + 0x04041036, + 0x00000804, + 0x4000047A, + 0x00000002, + 0x0400100C, + 0x3FC04804, + 0x40AB0440, + 0x2000007A, + 0x38000835, + 0x39207804, + 0x41E80050, + 0x00390018, + 0x04041036, + 0x00001804, + 0x4000047A, + 0x00000002, + 0x0400100C, + 0x3FC04804, + 0x40AB0440, + 0x2000007A, + 0x38000835, + 0x39207804, + 0x41E80050, + 0x00390028, + 0x04041036, + 0x00002804, + 0x4000047A, + 0x00000002, + 0x0400100C, + 0x3FC04804, + 0x40AB0440, + 0x2000007A, + 0x38000835, + 0x39207804, + 0x41E80050, + 0x00390038, +}; + +static gctUINT32 ppuMem3[] = +{ + 0x07841009, + 0x00200004, + 0xC0000000, + 0x200000D8, + 0xB8019005, + 0x0020D81C, + 0xC00106D0, + 0x0039004A, + 0x38011039, + 0x39200804, + 0xC0A90050, + 0x74000008, + 0xB8029005, + 0x0020D81C, + 0xC00106D0, + 0x0039004A, + 0x38021039, + 0x39202804, + 0xC0A90050, + 0x74000008, + 0xB8039005, + 0x0020D81C, + 0xC00106D0, + 0x0039004A, + 0x38031039, + 0x39204804, + 0xC0A90050, + 0x74000008, + 0xB8017005, + 0x002018FC, + 0x01C900C0, + 0x40390098, + 0xB8027005, + 0x002028FC, + 0x01C90140, + 0x40390098, + 0xB8037005, + 0x002038FC, + 0x01C901C0, + 0x40390098, + 0x04041036, + 0x00000804, + 0x4000047A, + 0x00000002, + 0x0400100C, + 0x3FC04804, + 0x40AB0440, + 0x2000007A, + 0x38000835, + 0x39207804, + 0x41E80050, + 0x00390018, + 0x04041036, + 0x00001804, + 0x4000047A, + 0x00000002, + 0x0400100C, + 0x3FC04804, + 0x40AB0440, + 0x2000007A, + 0x38000835, + 0x39207804, + 0x41E80050, + 0x00390028, + 0x04041036, + 0x00002804, + 0x4000047A, + 0x00000002, + 0x0400100C, + 0x3FC04804, + 0x40AB0440, + 0x2000007A, + 0x38000835, + 0x39207804, + 0x41E80050, + 0x00390038, +}; + +static gctUINT32 ppuMem4[] = +{ + 0x7E7E7E7E, + 0x797B7E7F, + 0x79797879, + 0x7D7C7B7A, + 0x80818181, + 0x7B7C7D7F, + 0x83817C79, + 0x7C7D8082, + 0x7C7C7D7D, + 0x7D7D7C7C, + 0x81818181, + 0x7D7D7F80, + 0x73777E83, + 0x71717171, + 0x7E7A7574, + 0x8A888782, + 0x7475797B, + 0x8D888078, + 0x8F92918F, + 0x787B8189, + 0x908E8781, + 0x787B838C, + 0x7A767576, + 0x85868581, + 0x817F8185, + 0x87838181, + 0x8182878D, + 0x787C8181, + 0x807A787A, + 0x8B878483, + 0x8D8A8787, + 0x8885878D, + 0x817E7D7D, + 0x81828483, + 0x7C7D7D7E, + 0x69686D76, + 0x6E6D6C6C, + 0x70727271, + 0x74727172, + 0x74757675, + 0x6F6E7072, + 0x70717170, + 0x6E6E7073, + 0x78767471, + 0x7A77726F, + 0x84817F7C, + 0x93949596, + 0x8D8D9092, + 0x8F8F8D88, + 0xB3A79991, + 0xABB4BABD, + 0x8D9199A2, + 0x77737274, + 0x676F787A, + 0x78716862, + 0x7577797A, + 0x74747474, + 0x78777675, + 0x78797A7A, + 0x74747576, + 0x817C7977, + 0x82848685, + 0x787A7D80, + 0x8886807A, + 0x79858D91, + 0x6D6A696E, + 0x676D7471, + 0x5659646A, + 0x3E3B3F45, + 0x44484A46, + 0x675B504A, + 0x82807A72, + 0x71777C80, + 0x6868696C, + 0x716E6D6B, + 0x7F7B7773, + 0x87909391, + 0x908F8A85, + 0x9596938F, + 0x89898C91, + 0x89878584, + 0x92908E8C, + 0x99999693, + 0x7B818993, + 0x7A818582, + 0x928B8078, + 0x928A8D97, + 0xA5968F93, + 0x939DA5AB, + 0x8F8D8A8C, + 0x8D8B8A8C, + 0x85878B8D, + 0x7C7D8082, + 0x7F807F7E, + 0x85858382, + 0x787B7F83, + 0x72717475, + 0x9B928378, + 0x9EB4B6AB, + 0x8C898589, + 0x998F8A8E, + 0x686D7E93, + 0x6A6A6865, + 0x65646568, + 0x64676868, + 0x60616162, + 0x7A7A7268, + 0x72767878, + 0x6C717577, + 0x716A6466, + 0x7D7E7B78, + 0x7A777679, + 0x8484817F, + 0x7A7E8183, + 0x89807E84, + 0x8080868D, + 0x60505069, + 0x7074857A, + 0x968D7A69, + 0x8E8C8D93, + 0x6D6D7A8D, + 0x8A7E7470, + 0x7C7D7F80, + 0x7A7A7A7B, + 0x7A7A7C7C, + 0x74757678, + 0x7C7A7775, + 0x75777A7C, + 0x7A7A7978, + 0x7A7A7A7A, + 0x81817C7A, + 0x7D7E7F81, + 0x8D898581, + 0x8185898D, + 0x7374787B, + 0x79797674, + 0x797A7A7B, + 0x79797878, + 0x807F7D7C, + 0x7D7E7F80, + 0x78797A7A, + 0x79797878, + 0x77787878, + 0x76767677, + 0x77777778, + 0x76767676, + 0x78767574, + 0x7F7E7C7A, + 0x797B7D7F, + 0x76767677, + 0x807E7C7A, + 0x82828181, + 0x7C7F8183, + 0x7979797A, + 0x84807B79, + 0x8E8D8C88, + 0x88888888, + 0x86878788, + 0x82818384, + 0x85868684, + 0x87878787, + 0x87878786, + 0x767B8185, + 0x72717173, + 0x7F818282, + 0x76787A7C, + 0x76767676, + 0x74747576, + 0x78736E6A, + 0x77787A7A, + 0x78757474, + 0x85827F7B, + 0x8D8E8D8D, + 0x8184878C, + 0x87837F7C, + 0x82848788, + 0x80818182, + 0x7E7E7F7F, + 0x8985817E, + 0x8C8C8D8C, + 0x8B8A8A8B, + 0x7A80878B, + 0x86817D7B, + 0x8D8D8C89, + 0x87888785, + 0x82828386, + 0x89848181, + 0x9293938F, + 0x8D8D8F92, + 0x868A8E8F, + 0x89837F7C, + 0x8C8E8F8D, + 0x78808689, + 0x63666A71, + 0x5F5C5F63, + 0x73726E67, + 0x75727274, + 0x68696E74, + 0x6C6B6C6B, + 0x83837D73, + 0x686B7074, + 0x807A716A, + 0x81828180, + 0x676C747C, + 0x68626060, + 0x7D7C7A72, + 0x7074787A, + 0x7F79716E, + 0x6E6A7585, + 0x686E7474, + 0x5C646665, + 0x4F4C4C52, + 0x63615A53, + 0x786F6763, + 0x7A7F8385, + 0x77757475, + 0x6B6E7171, + 0x65646467, + 0x6662605F, + 0x736E6A68, + 0x79797877, + 0x74757779, + 0x78747170, + 0x7B7D7D7B, + 0x7F7B7B7E, + 0x7C818483, + 0x8B878585, + 0x92939491, + 0x99949190, + 0x90959B9D, + 0x6C717E8A, + 0x7E7D7870, + 0x81828282, + 0x84828080, + 0x71727C87, + 0x7D7A7774, + 0x676F7E88, + 0x837B7068, + 0x84888A8B, + 0x7A7A7C80, + 0x76747373, + 0x76787979, + 0x6E6E7072, + 0x87817A73, + 0x8385878A, + 0x797B8082, + 0x8989837D, + 0x7E7C7D83, + 0x817B7A7B, + 0x757F8788, + 0x6F727476, + 0x6D6D6D6E, + 0x707A7A75, + 0x62626368, + 0x62636566, + 0x67656362, + 0x6E6E6E6E, + 0x7E78726E, + 0x817F7E7D, + 0x908D8A86, + 0x82818488, + 0x8A8C8B87, + 0x8184878A, + 0x807F7E7F, + 0x84858787, + 0x767A7F82, + 0x6C6C6C6C, + 0x7674716E, + 0x76787A7A, + 0x70717274, + 0x6E6D6E70, + 0x7D7B7772, + 0x807B7A78, + 0x878D8E87, + 0x79828989, + 0x74777774, + 0x4E545E67, + 0x6B645A50, + 0x8A796C68, + 0x63718490, + 0x7068615C, + 0x74747675, + 0x7C787471, + 0x82828180, + 0x7A7B7B7B, + 0x74757779, + 0x858B877E, + 0x7B747077, + 0x6F717980, + 0x74767572, + 0x6E6F6F6F, + 0x696A6B6D, + 0x565C6060, + 0x605F5A55, + 0x5F5F5E5D, + 0x66646160, + 0x736E6A68, + 0x7C7F7E7A, + 0x75757879, + 0x79797877, + 0x71717375, + 0x6C6E7273, + 0x6D6B6A69, + 0x71706F6E, + 0x66686A6B, + 0x61616263, + 0x7F7F7365, + 0x61697379, + 0x6B67625D, + 0x69696B6C, + 0x7776716B, + 0x75737274, + 0x6E788082, + 0x6F696365, + 0x6E6F7172, + 0x6E6D6B6B, + 0x74625E6F, + 0x74767679, + 0x77787979, + 0x7F7B7876, + 0x7C7D7D7C, + 0x6E71757A, + 0x63626262, + 0x6C6A6865, + 0x7C827C72, + 0x76747173, + 0x74706F6F, + 0x74777A78, + 0x6A6F7274, + 0x56595E64, + 0x615E775C, + 0x61625E6E, + 0x694F8559, + 0x13285176, + 0x1910261C, + 0x16201313, + 0x29191017, + 0x567B8219, + 0x836E967A, + 0x6B6C7087, + 0x726C6862, + 0x71827477, + 0x10171091, + 0x13252F10, + 0x1010171C, + 0x17101923, + 0x60847A58, + 0x66735684, + 0x6E6A6868, + 0x79777471, + 0x5C747B77, + 0x5676866C, + 0x5C67616B, + 0x102D1F14, + 0x67251028, + 0x955B7689, + 0x8A8B8B8C, + 0x8989898A, + 0x936E9489, + 0x79778177, + 0x102A6A6E, + 0x10151310, + 0x1E131214, + 0x13131F1D, + 0x817C8D8B, + 0x71706368, + 0x696E7479, + 0x7A746E69, + 0x7D807E7C, + 0x73737478, + 0x79797572, + 0x60646C74, + 0x7870596B, + 0x7B8D9A93, + 0x777A7274, + 0x105F6880, + 0x10181829, + 0x78604720, + 0x65696D6E, + 0x56585C60, + 0x8D685B55, + 0x8E78A599, + 0x21107F71, + 0x1024102F, + 0x10101B2D, + 0x13251618, + 0x634F5C16, + 0x44567A3C, + 0x5B565350, + 0x5D5D5D5C, + 0x5E5C5C5C, + 0x69686461, + 0x73716F6E, + 0x7A797775, + 0x56677099, + 0x465D624C, + 0x1F134859, + 0x10191910, + 0x10101010, + 0x10101010, + 0x71939610, + 0x8F574661, + 0x7A7C7B7A, + 0x7B797778, + 0x6068886D, + 0x705F7971, + 0x18101925, + 0x1F10101D, + 0x1B1A1F10, + 0x6C4C1F13, + 0x6866686A, + 0x5D545A6B, + 0x5A5A5959, + 0x57585959, + 0x6C625953, + 0x6D6F7171, + 0x44586863, + 0x4C4E3C58, + 0x101F2613, + 0x101D1110, + 0x161F1019, + 0x4F191310, + 0x6056617B, + 0x4A505F6C, + 0x855B3E60, + 0x3F4A3E78, + 0x107A3849, + 0x1B101A27, + 0x1910101A, + 0x102B101F, + 0x5E6F1112, + 0x67626263, + 0x65625E5C, + 0x5F626466, + 0x7A808588, + 0x4A556472, + 0x66629274, + 0x4E464873, + 0x6E673E56, + 0x102F6E57, + 0x1B101810, + 0x13131C19, + 0x101E1019, + 0x35685021, + 0x75676D6E, + 0x5981667E, + 0x463E4E4F, + 0x105C6350, + 0x19111822, + 0x21101E10, + 0x21232323, + 0x18191A1E, + 0x1D101E12, + 0x674E4810, + 0x66747368, + 0x7C6E5D59, + 0x6B6B6A6A, + 0x6F6E6B6A, + 0x7F80776E, + 0x74707077, + 0x51107274, + 0x10171E18, + 0x19191919, + 0x19191919, + 0x191B1C1C, + 0x17171819, + 0x646A6268, + 0x6A726C74, + 0x73637362, + 0x32485C51, + 0x2E1A1017, + 0x1F131023, + 0x2A103114, + 0x564A121C, + 0x4C45374F, + 0x35425A64, + 0x6D614E40, + 0x4550606C, + 0x3D454B50, + 0x282A2F35, + 0x7B797979, + 0x7D7F807E, + 0x7B7B7B7B, + 0x7F7E7D7C, + 0x80808180, + 0x7B7C7E7F, + 0x83817C79, + 0x7E7F8182, + 0x7A7A7A7B, + 0x7B7A7A7A, + 0x817F7D7C, + 0x83838281, + 0x777D7E7A, + 0x7B746E6E, + 0x807F7E7D, + 0x83838281, + 0x7A7A7D7F, + 0x918D857E, + 0x9192918F, + 0x7D81868D, + 0x8D8D8883, + 0x75798088, + 0x837D7977, + 0x8A8C8D89, + 0x84828285, + 0x87838183, + 0x7E81878D, + 0x81838280, + 0x8B878481, + 0x9393928F, + 0x8F939595, + 0x8787898C, + 0x82808080, + 0x83848684, + 0x82828283, + 0x6E6E737C, + 0x6F6E6E6E, + 0x72737371, + 0x75737272, + 0x7A7A7A79, + 0x72737476, + 0x70727373, + 0x6D6C6E71, + 0x7877746F, + 0x6F717475, + 0x8C847A72, + 0x8E8F9090, + 0x87888A8C, + 0x8A87837F, + 0xB1A5968D, + 0xA4ABB1B4, + 0x878C939B, + 0x78757578, + 0x6870787A, + 0x837A6E66, + 0x8B8B8B89, + 0x74747575, + 0x79787675, + 0x78797A7A, + 0x76767777, + 0x817D7B7A, + 0x7E818383, + 0x77747476, + 0x8C89847D, + 0x74818C91, + 0x6C68666A, + 0x6D70726A, + 0x5C5E666C, + 0x564A494C, + 0x6A707168, + 0x746E6866, + 0x81817D79, + 0x787B7E80, + 0x6E6F7174, + 0x7473706E, + 0x7F7B7875, + 0x878D8D8A, + 0x8D8D8985, + 0x99988F87, + 0x797D8591, + 0x85817E7C, + 0x93918D8A, + 0x98999894, + 0x7A7F8791, + 0x70747472, + 0x938B7D72, + 0x8D888C96, + 0x9C8F888D, + 0x979C9FA1, + 0x8E8D8E91, + 0x8D8C8C8D, + 0x87898C8E, + 0x7A7B7F82, + 0x8482807C, + 0x8181807E, + 0x74767B80, + 0x72727374, + 0x84817B75, + 0x949F9988, + 0x8D8B8687, + 0x938D898C, + 0x74717A8B, + 0x6B6D7074, + 0x65676869, + 0x68686765, + 0x5B5C6064, + 0x787A746A, + 0x78787474, + 0x787B7B7B, + 0x7C777374, + 0x84858381, + 0x807E7E81, + 0x8483817E, + 0x7C7F8283, + 0x877E7B7D, + 0x787E878B, + 0x524F4B5A, + 0x6661685E, + 0x948D7A6B, + 0x8D8B8C91, + 0x6E6A788C, + 0x7F7C7A77, + 0x7B7C7D7E, + 0x7B7B7B7B, + 0x7B7C7E7E, + 0x7777787A, + 0x74747576, + 0x76757474, + 0x7A797878, + 0x7A7A7A7A, + 0x807E7A78, + 0x7B7C7D7F, + 0x87868381, + 0x81838687, + 0x7374797D, + 0x7A7A7774, + 0x7B7C7D7D, + 0x7A7A7A7B, + 0x7F7E7D7D, + 0x7C7C7E7F, + 0x78797A7A, + 0x78787878, + 0x77787878, + 0x76767677, + 0x74747575, + 0x74747474, + 0x77747473, + 0x7D7C7A79, + 0x787A7B7C, + 0x77767677, + 0x7D7D7D7C, + 0x81807E7D, + 0x80818283, + 0x7B7B7C7E, + 0x8683817F, + 0x8B8B8A88, + 0x88888787, + 0x85868787, + 0x81818183, + 0x85868684, + 0x86878787, + 0x85858585, + 0x787C8082, + 0x75747475, + 0x7F7F7F7F, + 0x7C7D7E7F, + 0x7F7F7E7E, + 0x7C7C7E7F, + 0x7A797472, + 0x7375797B, + 0x7A787675, + 0x8583817D, + 0x8B8C8C8B, + 0x7F818588, + 0x87837E7B, + 0x81848788, + 0x7E808181, + 0x81807F7E, + 0x8784817E, + 0x8B8C8B8A, + 0x8A8A8A8B, + 0x7A7F8689, + 0x85817E7C, + 0x89898887, + 0x8184888B, + 0x87868281, + 0x88858281, + 0x8D8D8D8B, + 0x8B8B8C8D, + 0x86888C8D, + 0x87838180, + 0x83868989, + 0x797E8285, + 0x6C6E6F74, + 0x5F5C6064, + 0x73726E66, + 0x76757475, + 0x62636973, + 0x6E696763, + 0x8A8A8378, + 0x646E7A82, + 0x79726862, + 0x787A7978, + 0x5E636B74, + 0x615B5A5C, + 0x7273726B, + 0x7C7C7A78, + 0x817E7B7B, + 0x6D6A7584, + 0x686C7172, + 0x61636668, + 0x5053585D, + 0x605B5047, + 0x776F6662, + 0x7F7F7C7A, + 0x75777A7D, + 0x6B6E6F6E, + 0x65646568, + 0x68666564, + 0x736F6B68, + 0x7A797876, + 0x7B7C7C7B, + 0x70707375, + 0x7A797673, + 0x7F7A7B7E, + 0x7E838785, + 0x8B878585, + 0x9092928F, + 0x99939190, + 0x8E949A9C, + 0x72757D85, + 0x807F7C76, + 0x81807F7F, + 0x86868482, + 0x70717B87, + 0x7A787573, + 0x6C717B83, + 0x8A83796F, + 0x85898C8D, + 0x7A7A7D81, + 0x74747677, + 0x6F727474, + 0x6E6C6E6F, + 0x89847A73, + 0x82848789, + 0x787A7E81, + 0x83817A73, + 0x7A7A7A80, + 0x817A7979, + 0x77808788, + 0x70737475, + 0x6D6D6E6E, + 0x6E777773, + 0x62626266, + 0x5F606262, + 0x63626160, + 0x6B6B6B6A, + 0x817A726D, + 0x797B8185, + 0x8D88817B, + 0x81818589, + 0x8E8E8C87, + 0x8286898C, + 0x81818181, + 0x80818284, + 0x73767B7F, + 0x6C6B6C6C, + 0x7574716E, + 0x7678797A, + 0x71727374, + 0x6A6A6D6F, + 0x7A78746E, + 0x7877797A, + 0x8B8E8B81, + 0x7A818685, + 0x777A7A77, + 0x4C505A62, + 0x6F695D52, + 0x87786D69, + 0x6571838D, + 0x746F6761, + 0x74747677, + 0x7A777473, + 0x7A7B7B7B, + 0x77767575, + 0x7A7A7A79, + 0x868B8781, + 0x7E79767C, + 0x6F71797F, + 0x74757572, + 0x6E6F7070, + 0x696A6B6D, + 0x545E6363, + 0x53514F4E, + 0x5E5D5C5A, + 0x5D5D5D5E, + 0x6A64615F, + 0x777B7B74, + 0x6C6E7174, + 0x696B6D6D, + 0x6C6A6C6E, + 0x6C6E6F6E, + 0x6A686868, + 0x6E6E6D6C, + 0x696C6E6E, + 0x64646668, + 0x7B7B7165, + 0x61687076, + 0x6A67625D, + 0x62646769, + 0x716C655F, + 0x76747373, + 0x71797C7D, + 0x6D69676A, + 0x7072716F, + 0x6F6E6C6E, + 0x74625D6E, + 0x76787779, + 0x77767676, + 0x79797877, + 0x7A7B7C7C, + 0x6F727478, + 0x71706F6F, + 0x79787573, + 0x7B7A7977, + 0x6D6F7479, + 0x6E6E7072, + 0x7875726F, + 0x6D717475, + 0x5B5D6268, + 0x615C6759, + 0x61685E6E, + 0x7A7D5C44, + 0xD6144E6D, + 0xDDE5E2EB, + 0xE4EBEBEB, + 0xEBDCE3EB, + 0x8F6E73DD, + 0x64518573, + 0x6766626E, + 0x6E676260, + 0x74887C71, + 0xE7DD3B61, + 0xE7E9CCEB, + 0xDCE1EBEB, + 0xEBDEDCE2, + 0x606F8774, + 0x6371687F, + 0x6D6A6868, + 0x7674726F, + 0x80817C79, + 0x61748786, + 0x78635A4E, + 0xEBE5D014, + 0x86C8E5E7, + 0x6E748F7B, + 0x77777778, + 0x75757676, + 0x7E647F74, + 0x7470656D, + 0xE1444A68, + 0xE9CED7EB, + 0xE6DDDDDD, + 0xE7EAE9E8, + 0x874A7282, + 0x9F7E876E, + 0x696E7478, + 0x79746E69, + 0x7C808180, + 0x6E6E7076, + 0x6B6E6E6F, + 0x5E606267, + 0x5F5C646D, + 0x6E76716E, + 0x73796A73, + 0x17496762, + 0xDEEBDFE4, + 0x899AC9D7, + 0x6267696B, + 0x56575A5E, + 0x6B5C656F, + 0x83687A6F, + 0xD31A7483, + 0xEAD3E5E7, + 0xEBEBDEE0, + 0xEBE4EBDF, + 0x692F63E8, + 0x534A584E, + 0x58565554, + 0x5C5C5B5A, + 0x5E5C5C5C, + 0x65646260, + 0x6B696868, + 0x70706E6D, + 0x7A707474, + 0x4E4A543B, + 0xC41B564C, + 0xE9E5EBC4, + 0xE8E8E8E8, + 0xE8E8E8E8, + 0x8F556FEB, + 0x6E7A7593, + 0x5B5F6E7B, + 0x61646560, + 0x4A727E51, + 0x6B54585E, + 0xE1EBD52E, + 0xE4EBE7E8, + 0xE6E8EBEA, + 0x6668E6EB, + 0x4E5D5E4F, + 0x6E6C5F4F, + 0x53535252, + 0x53535454, + 0x635C5653, + 0x67686867, + 0x375B6352, + 0x44505A4F, + 0xEBE2D820, + 0xEAE6D6CC, + 0xDCE8E5E8, + 0x6FE8E2EB, + 0x45556166, + 0x43504E43, + 0x495A525A, + 0x4E4D4276, + 0x2B446056, + 0xD0EBEBDC, + 0xDCE4EBD1, + 0xEBE2D2EB, + 0x438CE3D5, + 0x60626E6E, + 0x625E5956, + 0x5D606264, + 0x74736F6C, + 0x5E626970, + 0x6E6C8561, + 0x4A4B496C, + 0x7A63582A, + 0xE929566C, + 0xDDD9EBE9, + 0xE2EBE2EB, + 0xE7DEDBEB, + 0x785A54DC, + 0x7B74715E, + 0x687B5673, + 0x443E5154, + 0x1462654E, + 0xEBE3E3E4, + 0xEBE3EBE7, + 0xE2E3E3E3, + 0xDFDFE0E1, + 0xE8E0E9E2, + 0x636C60C8, + 0x727B786E, + 0x7C746867, + 0x58575D62, + 0x7D776C60, + 0x7C7C746D, + 0x6B6B6E75, + 0xD0347668, + 0xEBEBE4CA, + 0xE9E9E9E9, + 0xE9E9E9E9, + 0xE9E9E9E9, + 0xE8E8E8E8, + 0x5B5D5756, + 0x61646062, + 0x50485064, + 0x104D4946, + 0xDFE9EAE8, + 0xEBEBEADB, + 0xD7EADEEB, + 0x4876D4E3, + 0x5B4E4958, + 0x3F4B616D, + 0x6960534A, + 0x4E56626A, + 0x3F474D50, + 0x2C2E3238, + 0x79747372, + 0x8483817E, + 0x7F808181, + 0x807F7F7F, + 0x80818080, + 0x7D7E7F80, + 0x82817C79, + 0x80808182, + 0x77777878, + 0x78787777, + 0x817C7A77, + 0x8B8A8785, + 0x7A817C74, + 0x877A6D6F, + 0x82858789, + 0x7C7D7E81, + 0x81818284, + 0x95918A84, + 0x93928F8D, + 0x87898E92, + 0x8A8D8C89, + 0x75787D84, + 0x938A8380, + 0x999B9C99, + 0x898A8A8B, + 0x89848286, + 0x7E81888F, + 0x8C8A8681, + 0x9699958F, + 0x999E9E98, + 0x8F9B9F9D, + 0x84888A89, + 0x83828283, + 0x87878786, + 0x8A8A8989, + 0x74747A84, + 0x706E6F71, + 0x74747472, + 0x79747373, + 0x8383817D, + 0x787A7C7F, + 0x6F727476, + 0x68686B6E, + 0x7876726D, + 0x686D747A, + 0x9387776B, + 0x898B8B8C, + 0x82848687, + 0x84837F7A, + 0xA59A8D85, + 0x999FA3A5, + 0x83868C92, + 0x7A797A7D, + 0x6E747A7C, + 0x8A837A73, + 0x8A8B8D8D, + 0x74757778, + 0x79787674, + 0x78787878, + 0x78787878, + 0x817E7C7B, + 0x7F818383, + 0x7A777778, + 0x8081817F, + 0x6B788287, + 0x68635F62, + 0x73757066, + 0x65686C6E, + 0x75656062, + 0x8B929389, + 0x7D7E8081, + 0x7D7D7C7C, + 0x81818180, + 0x76787B7F, + 0x79787674, + 0x7D7B7A79, + 0x84868581, + 0x8B8B8884, + 0x8E938C83, + 0x6C6D7481, + 0x817D7A78, + 0x8D8C8885, + 0x93949391, + 0x8183878E, + 0x6E6F706F, + 0x8D898074, + 0x8C898D96, + 0x8D838188, + 0x9D9C9894, + 0x8C8F9399, + 0x8A898887, + 0x8286898A, + 0x787C8389, + 0x7C7A7977, + 0x7E7E7B7A, + 0x6F73777C, + 0x78726F6F, + 0x8182827F, + 0x8F958E81, + 0x8E8D8887, + 0x8B8B8989, + 0x7F787882, + 0x6B6E757B, + 0x6A6C6E6C, + 0x686A6867, + 0x56585D63, + 0x777A736A, + 0x7D7A7473, + 0x8686827F, + 0x87868484, + 0x88898887, + 0x83838487, + 0x85838180, + 0x7D7E8184, + 0x81818180, + 0x5E6C7B81, + 0x49514A4C, + 0x6259574A, + 0x938D7E71, + 0x8C8A8B8F, + 0x736A788D, + 0x747C8481, + 0x7A7A7B7B, + 0x7D7C7B7B, + 0x7D7E7F80, + 0x797A7A7B, + 0x6B6E7477, + 0x77746E6B, + 0x79787777, + 0x7979797A, + 0x7E7C7975, + 0x7A7A7A7C, + 0x81818282, + 0x82828181, + 0x74767B80, + 0x7D7B7975, + 0x80818181, + 0x7C7C7E7F, + 0x7E7E7D7D, + 0x7A7B7C7D, + 0x78797A7B, + 0x76767677, + 0x77777777, + 0x76767676, + 0x71727272, + 0x70707171, + 0x7472706F, + 0x7A797875, + 0x77797A7A, + 0x76767676, + 0x7A7A7B7B, + 0x7F7C7A78, + 0x84838382, + 0x80818283, + 0x87878685, + 0x87878787, + 0x87878786, + 0x85858687, + 0x81808181, + 0x86878684, + 0x85868787, + 0x82828384, + 0x797A7C7D, + 0x76767778, + 0x7F7D7A7A, + 0x84838281, + 0x87878786, + 0x84858787, + 0x80817F7E, + 0x6E72787D, + 0x7F7C7A79, + 0x86858381, + 0x86878787, + 0x7A7C8083, + 0x86817C79, + 0x81838787, + 0x7E818284, + 0x7D7C7C7C, + 0x8581807E, + 0x8A8A8887, + 0x88888989, + 0x7B7F8487, + 0x8381807F, + 0x83848484, + 0x7E7F8488, + 0x81828280, + 0x8784817E, + 0x87878687, + 0x87878786, + 0x87878788, + 0x87878686, + 0x7A7E8387, + 0x777A7D7F, + 0x75757475, + 0x5F5E6266, + 0x72706C65, + 0x78797675, + 0x5D5E6470, + 0x6C65605C, + 0x89898376, + 0x6874818A, + 0x6E6A6462, + 0x71716E6D, + 0x5F63696E, + 0x5A55575C, + 0x61656863, + 0x867F756E, + 0x84868788, + 0x6E6E7884, + 0x686C7071, + 0x6865676B, + 0x5761696C, + 0x5C564A3E, + 0x7068605C, + 0x837F7872, + 0x73767B81, + 0x6B6E6F6F, + 0x65646568, + 0x6F6F6E6E, + 0x7573706F, + 0x7E7A7775, + 0x86858381, + 0x686E787F, + 0x7C78706A, + 0x8078797C, + 0x878D8F8A, + 0x8B878686, + 0x8D8F8F8E, + 0x96928F8E, + 0x8B909799, + 0x7A7A7D81, + 0x82817F7C, + 0x84807E7F, + 0x8D8E8E8A, + 0x6F717A86, + 0x79777473, + 0x7474787C, + 0x928D847A, + 0x85898D8E, + 0x7A7B7E81, + 0x6E717477, + 0x68696C6E, + 0x69656464, + 0x89837A71, + 0x8386888A, + 0x787A7F81, + 0x7C79716B, + 0x7676787A, + 0x837C7979, + 0x79808788, + 0x72747677, + 0x6B6C6E6F, + 0x6C73736E, + 0x62626263, + 0x5C5D5E5F, + 0x5F5E5C5C, + 0x62606060, + 0x7D766D65, + 0x6E72797E, + 0x83807972, + 0x7F7F8489, + 0x918F8A84, + 0x81848788, + 0x81818080, + 0x7D7F8181, + 0x7274797C, + 0x6C6C6C6C, + 0x7473706E, + 0x76777878, + 0x73747475, + 0x66686C6F, + 0x76746E69, + 0x77747575, + 0x8A8D8B81, + 0x83878786, + 0x80828381, + 0x5457626B, + 0x6E6A6259, + 0x83766E6E, + 0x67718189, + 0x7A786F68, + 0x75747579, + 0x76767474, + 0x72737476, + 0x74716F6E, + 0x83817D78, + 0x878A8886, + 0x82818183, + 0x6F71787E, + 0x73747573, + 0x6E6E6F70, + 0x69696B6C, + 0x525C6261, + 0x4D4D4B4B, + 0x5D5C5B59, + 0x56585B5C, + 0x615A5858, + 0x71797A6E, + 0x6262676B, + 0x5E626564, + 0x625F6061, + 0x6B6A6865, + 0x6B696868, + 0x6F6E6E6D, + 0x6E707172, + 0x68686A6C, + 0x76756E65, + 0x62676D72, + 0x6867625D, + 0x5B5D6266, + 0x6F69625E, + 0x70727373, + 0x77797774, + 0x6A6A6D71, + 0x7576726E, + 0x6F6E6E71, + 0x73625D6B, + 0x76787777, + 0x75727272, + 0x787A7B7A, + 0x78797A7A, + 0x72737476, + 0x78767676, + 0x7E7D7B7A, + 0x80787B85, + 0x626B7A83, + 0x6A6A6968, + 0x7974706C, + 0x71747778, + 0x6264686C, + 0x6A645A5F, + 0x50625B72, + 0x58896E95, + 0xE9162710, + 0xE8E1EBEB, + 0xE4EBE2E6, + 0xE1DFEBE2, + 0x101028E7, + 0x68454A10, + 0x72625669, + 0x7C776C6A, + 0x5174806E, + 0xD8D12810, + 0xE8EBCAEB, + 0xE4EAEBDE, + 0xEBE5E2D8, + 0x87645811, + 0x7363697C, + 0x6B6A6969, + 0x706F6E6C, + 0x6E768768, + 0x7A78928C, + 0x94726577, + 0xEBD2E015, + 0x5CD4EBE0, + 0x6F8D9355, + 0x6C6C6C6C, + 0x6B6B6B6B, + 0x797A7D63, + 0x1F1C1050, + 0xD0121326, + 0xEBEBEBE7, + 0xE8EBEBEB, + 0xE5EBDAE3, + 0x552E2519, + 0x8770725F, + 0x676A7074, + 0x74706A66, + 0x777B7D7D, + 0x696A6D72, + 0x5E646B6F, + 0x615F5C5B, + 0x6C5C6A5E, + 0x82857A7A, + 0x605F5B68, + 0x3972666E, + 0xEAEBE5EA, + 0x7A92D6E6, + 0x5E626365, + 0x5456575B, + 0x7058504F, + 0x7C727179, + 0xE21B2B10, + 0xEBD2E8E5, + 0xE0E8E6E8, + 0xE5E5D0D7, + 0x633438DA, + 0x63505362, + 0x56545454, + 0x5C5B5957, + 0x5D5C5C5B, + 0x5F5F5F5E, + 0x61605F5E, + 0x63636262, + 0x8B636352, + 0x635A6349, + 0xDE103B15, + 0xEBE4E4EB, + 0xEBEBEBEB, + 0xEBEBEBEB, + 0x1A132DCC, + 0x70552812, + 0x4E5C6D77, + 0x7D6E594C, + 0x146E8866, + 0x1A101030, + 0xD6E7E33E, + 0xE8E2DEEB, + 0xEBE6DDCC, + 0x1728D6EB, + 0x5B52341A, + 0x56443841, + 0x4D4D4C4C, + 0x504F4F4E, + 0x56555353, + 0x5E5D5C59, + 0x363F3B44, + 0x1013665B, + 0xEBEAE531, + 0xEBE0E4E8, + 0xE5E2EBEB, + 0x32DDD6EB, + 0x6256392C, + 0x4B413844, + 0x394E525A, + 0x5A4A4D70, + 0x25191F2A, + 0xE0EBEBDF, + 0xE3E7EBE4, + 0xEBE2E4EB, + 0x1434CDEB, + 0x6F7B7D53, + 0x645E5853, + 0x5F626567, + 0x716D635C, + 0x6C6C6D70, + 0x66686C4E, + 0x5860505B, + 0x78637D76, + 0xDE2C1E15, + 0xE8E3EBE8, + 0xE5E5E5E9, + 0xEBEBEBE2, + 0x10112DEB, + 0x6F4D3510, + 0x7A856379, + 0x3E435E67, + 0x10555642, + 0xEBEAEBEB, + 0xE3E4E8EA, + 0xE2E2E2E2, + 0xE3E3E3E3, + 0xE4DEE6EB, + 0x7D7926DD, + 0x7E817A72, + 0x77767577, + 0x50525D67, + 0x7F77695A, + 0x7A79746F, + 0x65686E75, + 0xE5295C55, + 0xEBE7EBE8, + 0xE6E6E6E6, + 0xE6E6E6E6, + 0xE8E8E8E8, + 0xEAEAE9E9, + 0x5352534C, + 0x5151544D, + 0x26194F48, + 0x1114294F, + 0xEBDDD9EB, + 0xE6E4E8EB, + 0xD6E9E1EB, + 0x104ABBE9, + 0x503A4345, + 0x66676866, + 0x65605B59, + 0x565C6568, + 0x40464B4F, + 0x3133363A, + 0x78737070, + 0x8887847F, + 0x83868788, + 0x7E7F8081, + 0x81818181, + 0x7F7F8081, + 0x817F7B79, + 0x80808081, + 0x75767777, + 0x77777675, + 0x827D7976, + 0x8F8E8B87, + 0x7A7E7A74, + 0x8B817574, + 0x85898D8E, + 0x7A7B7D81, + 0x81818284, + 0x93908A85, + 0x938E8987, + 0x91939696, + 0x878C8E8F, + 0x787A7D81, + 0x988F8783, + 0xA4A4A39F, + 0x8E939699, + 0x8C868388, + 0x83878D93, + 0x8F8D8985, + 0x90999A93, + 0x8F98988F, + 0x8F9D9C91, + 0x82878786, + 0x85838485, + 0x88898887, + 0x8E8E8D8D, + 0x78797F88, + 0x72707273, + 0x77787774, + 0x7C777575, + 0x8B8A8782, + 0x7E818689, + 0x6D70757A, + 0x62626568, + 0x76736E68, + 0x6E707477, + 0x92887C72, + 0x8B8C8D8D, + 0x84858788, + 0x82868581, + 0x9088807E, + 0x8F929394, + 0x8385888C, + 0x7E7D8082, + 0x757A7F80, + 0x8A87817D, + 0x80818588, + 0x7476797A, + 0x77767474, + 0x77777676, + 0x79797878, + 0x7E7B7A7A, + 0x81828281, + 0x81818182, + 0x6E747A80, + 0x6871787A, + 0x6B666262, + 0x7879726C, + 0x6E767674, + 0x81797474, + 0x878C8D8A, + 0x7E82878A, + 0x7C7B7A7B, + 0x87858280, + 0x7A7E8286, + 0x7B7C7A7A, + 0x7A7A797A, + 0x81807E7B, + 0x8C8B8784, + 0x7B898D8C, + 0x6F68636B, + 0x80808181, + 0x7F7F7F80, + 0x8B898582, + 0x92908D8D, + 0x8F919598, + 0x8E959993, + 0x8C8A8C92, + 0x807A7C87, + 0xA19B9189, + 0x888E989F, + 0x8B8A8782, + 0x7D848A8C, + 0x757A8186, + 0x75757474, + 0x81817F7D, + 0x71747A7E, + 0x7A716D6B, + 0x81868884, + 0x767A7A77, + 0x82817C76, + 0x86898A8B, + 0x817C7A7F, + 0x6E707476, + 0x7474726F, + 0x686D6F6F, + 0x5F5D5E62, + 0x7574726F, + 0x7D7A7775, + 0x8D8C8780, + 0x888B8C8C, + 0x8C8B8A89, + 0x8386888B, + 0x85828182, + 0x7D7D8084, + 0x7A818481, + 0x50627276, + 0x55605A59, + 0x726F6E5A, + 0x8D8A7E74, + 0x8786868A, + 0x786F7A8C, + 0x737D8887, + 0x7A7A7A7A, + 0x7E7D7B7A, + 0x7C7E7E7F, + 0x797A7A7B, + 0x686D7377, + 0x77736D68, + 0x78787776, + 0x78787979, + 0x7C7B7874, + 0x79797A7B, + 0x7B7D8183, + 0x83817D7B, + 0x76797E82, + 0x807E7A77, + 0x82838383, + 0x7D7E8081, + 0x7B7C7C7C, + 0x7A7A7A7B, + 0x77797A7B, + 0x75757576, + 0x76767676, + 0x76767676, + 0x6F6F6F6F, + 0x6E6E6E6E, + 0x706E6D6C, + 0x75747472, + 0x76777979, + 0x73737474, + 0x76777574, + 0x7A787574, + 0x87858280, + 0x82848787, + 0x85868687, + 0x82838384, + 0x86858584, + 0x84848586, + 0x81808181, + 0x87878785, + 0x84858687, + 0x7E7F8182, + 0x79797979, + 0x76777879, + 0x7F7B7977, + 0x86858481, + 0x88878787, + 0x85868787, + 0x85868583, + 0x72757B81, + 0x8481807E, + 0x86868786, + 0x7F818283, + 0x7577797C, + 0x85807A76, + 0x80828587, + 0x7F828486, + 0x7274777B, + 0x81807E7D, + 0x88878684, + 0x87878787, + 0x7D808285, + 0x8181807F, + 0x7D7E8081, + 0x837F7E80, + 0x757C8487, + 0x84817B76, + 0x82828384, + 0x83828180, + 0x87868584, + 0x84858686, + 0x70757C81, + 0x72747577, + 0x79777473, + 0x61616468, + 0x716E6A65, + 0x7B7B7774, + 0x5F5F6572, + 0x6462615F, + 0x7A7A756C, + 0x777A7B7C, + 0x686A6E74, + 0x746F6B68, + 0x6E717475, + 0x615D6269, + 0x5E666C69, + 0x81766961, + 0x85878988, + 0x6F737B84, + 0x6B6D6F6F, + 0x6E69696D, + 0x68707675, + 0x63635B52, + 0x6C645D5E, + 0x7E7C7874, + 0x7374787B, + 0x6A6F7476, + 0x66646265, + 0x7576736E, + 0x73747474, + 0x807B7875, + 0x8E8D8985, + 0x6972808A, + 0x847D736A, + 0x7D787A80, + 0x80878B87, + 0x8C8A8887, + 0x8C8D8D8D, + 0x928D8C8B, + 0x858B9294, + 0x7B7B7B7C, + 0x7F7E7D7B, + 0x837C7A7A, + 0x8F92928D, + 0x6F707983, + 0x79777473, + 0x7C777677, + 0x94928C83, + 0x82878B8D, + 0x797A7C80, + 0x6D6F7477, + 0x68686A6C, + 0x6E67625F, + 0x87847E76, + 0x82838687, + 0x7A7C7F81, + 0x7474716F, + 0x73737374, + 0x857F7A78, + 0x7C818788, + 0x74787A7B, + 0x66686C70, + 0x6A706F69, + 0x64646264, + 0x5C5D5E5F, + 0x5A5A5A5B, + 0x54515152, + 0x736C6259, + 0x6B686B6E, + 0x7C7A7771, + 0x7A7B8084, + 0x8A87837E, + 0x7B7D7F80, + 0x7C7B7A7A, + 0x77797A7C, + 0x6E707476, + 0x6D6C6C6B, + 0x7473716E, + 0x76767676, + 0x75757676, + 0x63666C71, + 0x726F6A66, + 0x827B7774, + 0x8990938C, + 0x91918F8D, + 0x8C8E9091, + 0x6E737C85, + 0x73747470, + 0x81777476, + 0x67707D85, + 0x7F7D746D, + 0x7674747A, + 0x74747575, + 0x6E6E7173, + 0x736F6D6C, + 0x88857F79, + 0x87878788, + 0x86878989, + 0x7071777D, + 0x70737473, + 0x6B6D6E6E, + 0x6868696A, + 0x5A5D6162, + 0x6264625C, + 0x5C5C5C5C, + 0x56585B5C, + 0x5C565657, + 0x6C76796B, + 0x5C595C5F, + 0x61646662, + 0x59565656, + 0x6866625D, + 0x6F6E6D6D, + 0x74737271, + 0x70727374, + 0x696A6C6E, + 0x6E6E6A65, + 0x6265686C, + 0x6868625E, + 0x585C6066, + 0x6C645E5B, + 0x6C6E7271, + 0x7A78726D, + 0x6B6E7278, + 0x7B7B746E, + 0x6E6D6F75, + 0x71625E6A, + 0x73757474, + 0x726D6C6E, + 0x7C7F7F7A, + 0x74767779, + 0x74747474, + 0x74737372, + 0x78777674, + 0x897F8391, + 0x636B7D8D, + 0x6D66615E, + 0x75767673, + 0x73767879, + 0x696A6D70, + 0x605E4C65, + 0x51585367, + 0x4F5C4E45, + 0xE2E4D342, + 0xEBEBE8DC, + 0xE8E7E2EB, + 0xC8D7EBE2, + 0xEBEBD0EB, + 0x7B8CDFDE, + 0x78706471, + 0x80816A68, + 0x28567456, + 0xE2EBEAEB, + 0xD7EBEBD8, + 0xEBE4E6E3, + 0xE5E5EBDF, + 0x6858CAE6, + 0x6D657D65, + 0x6868696A, + 0x6A696968, + 0x6E5C6D65, + 0x96949381, + 0x70687C93, + 0xE5EBDB32, + 0x63C4D4CA, + 0x6A767A63, + 0x6B6B6B6B, + 0x6B6B6B6B, + 0x716E6E55, + 0xEBB43E68, + 0xEBC6E7EB, + 0xC7DFEBE5, + 0xEBEBE8E4, + 0xD6EBDEEB, + 0x62EBE1D6, + 0x767A536E, + 0x62666C70, + 0x6E6B6662, + 0x70717171, + 0x6E6E6E6F, + 0x5C636E74, + 0x67635E5B, + 0x5D596D67, + 0x65636463, + 0x68625B71, + 0x15535C71, + 0xDBE4E1EA, + 0x828FD7DA, + 0x575A5B5C, + 0x51525456, + 0x695D5757, + 0x686B4A69, + 0xEBE0BB3A, + 0xE8E9DEE6, + 0xE9E9E6EB, + 0xE1EBEBEB, + 0x69DCCFEB, + 0x4B3E444A, + 0x5451504F, + 0x5C5A5856, + 0x5C5C5B5B, + 0x58595B5C, + 0x58585857, + 0x5A595959, + 0x5046546A, + 0x4950363A, + 0xE5EBE110, + 0xC5EBD4E6, + 0xE7E7E7E7, + 0xE7E7E7E7, + 0xEBEBD3E7, + 0x6475D6E0, + 0x64635B53, + 0x7C716562, + 0x1E457A8E, + 0xDFEBEBC5, + 0xE3E3EBC7, + 0xEBE6E5EB, + 0xEBE8E5DB, + 0xEBE0EBDC, + 0x40125DE0, + 0x3E486060, + 0x4B4B4B4B, + 0x4E4D4C4C, + 0x4D4F5154, + 0x5654504E, + 0x4E6D4C4D, + 0xDF392F31, + 0xC3E4E0E3, + 0xE0DEEBEB, + 0xEBE1E2DC, + 0xE1EBE7D2, + 0x4A165EE8, + 0x45536868, + 0x4D4E535C, + 0x3631524E, + 0xD7DC2E3E, + 0xEBCBDCEB, + 0xE6E7E5EB, + 0xC3E4EBEB, + 0xEBEBDAEB, + 0x7557466E, + 0x6E68615C, + 0x66696E70, + 0x6F6E6863, + 0x5F62666C, + 0x585B5B55, + 0x747C6756, + 0x5B495256, + 0xE1EBC329, + 0xEBEBD2C9, + 0xEBE2EBEB, + 0xD4DCEBE8, + 0xEBEBC5EB, + 0x5A6AC1E2, + 0x8D96746E, + 0x474D6B74, + 0x145F5E4A, + 0xE8E4EBE9, + 0xEBEBE0EB, + 0xE7E7E8E8, + 0xE7E8E8E8, + 0xEBEBE0E7, + 0x6E7742EB, + 0x807C7671, + 0x6C71797E, + 0x67696E70, + 0x726E6A67, + 0x71707071, + 0x62676D70, + 0xE7255C5E, + 0xE6E1EBEB, + 0xEBEBEBEB, + 0xEBEBEBEB, + 0xEBEBEBEB, + 0xEBEBEBEB, + 0x4B4A5751, + 0x3A404C3E, + 0xCE453E49, + 0xE6E3DDE5, + 0xD9E4E9EB, + 0xDCEBEBD9, + 0xE1EBEBCA, + 0xD1DDEBEB, + 0x5043574F, + 0x6E6B6861, + 0x63606162, + 0x585E6466, + 0x3D414548, + 0x32333538, + 0x7B757372, + 0x8A898781, + 0x85888C8D, + 0x7C7D7F81, + 0x82828282, + 0x7F808181, + 0x7D7D7A79, + 0x7F7D7C7D, + 0x77787979, + 0x79797877, + 0x85807C7A, + 0x908F8D89, + 0x78777A7F, + 0x8786827D, + 0x86898C8D, + 0x7D7E8082, + 0x7A7A7B7D, + 0x8886817B, + 0x90888380, + 0x96999997, + 0x82878B8E, + 0x7A7B7C7F, + 0x8B85807D, + 0x9D9A9691, + 0x8F969EA4, + 0x8D878488, + 0x8C8E9294, + 0x8C8C8C8B, + 0x818D918D, + 0x878D8980, + 0x8E99917F, + 0x85888785, + 0x86848585, + 0x88898A88, + 0x8C8C8A8B, + 0x76767C86, + 0x74737374, + 0x7A7B7A78, + 0x7D7A7878, + 0x8D8C8883, + 0x81868B8E, + 0x696E747B, + 0x5C5C6063, + 0x736F6862, + 0x7C797471, + 0x8B888480, + 0x8E909191, + 0x88898B8D, + 0x858B8B88, + 0x7F7A787C, + 0x89888787, + 0x87878888, + 0x81818284, + 0x7C7F8283, + 0x8786817E, + 0x84848687, + 0x76787A7C, + 0x76757474, + 0x76767676, + 0x77777777, + 0x78777778, + 0x7A7A7B7A, + 0x8684807D, + 0x70757E84, + 0x71747574, + 0x77736E6E, + 0x7C7D7879, + 0x76858279, + 0x7A7A7673, + 0x7A767475, + 0x7A7D8182, + 0x7E7C7A7A, + 0x8583817F, + 0x7D7F8184, + 0x7C7E7F7E, + 0x7877777A, + 0x817C7A7A, + 0x8F8D8986, + 0x7A8C9598, + 0x8074686B, + 0x83888D8E, + 0x7174787D, + 0x837C7673, + 0xA49E948B, + 0xBABDC4C9, + 0x96A6B6BA, + 0x89868284, + 0x7A787C86, + 0xA1998D83, + 0x878D98A0, + 0x9B99938C, + 0x848E999B, + 0x7474777A, + 0x79787775, + 0x7D7E7D7C, + 0x6F72767A, + 0x706B696B, + 0x6F747876, + 0x706A6A6A, + 0x8A8C877B, + 0x84898D8E, + 0x7C7F8181, + 0x78787470, + 0x7D7A7575, + 0x69717679, + 0x7A736966, + 0x6D6F7A84, + 0x7C7B7872, + 0x8C8D8881, + 0x82878B8B, + 0x8C888787, + 0x80858B8D, + 0x817C7A7C, + 0x807D7E82, + 0x767C7F7B, + 0x66717875, + 0x6E747377, + 0x818D967E, + 0x88877F78, + 0x83818084, + 0x7A757B86, + 0x757C8482, + 0x7979797A, + 0x7C7B7A7A, + 0x7A7B7B7C, + 0x77787879, + 0x6D6F7375, + 0x75736F6D, + 0x78787776, + 0x78787979, + 0x7E7C7976, + 0x7A7A7B7D, + 0x7A7D8184, + 0x84817D7A, + 0x797B8084, + 0x82817E7A, + 0x82838485, + 0x7C7D7F81, + 0x7A7A7A7B, + 0x7979797A, + 0x76787A7A, + 0x74747475, + 0x76767675, + 0x76767676, + 0x6F6F6F6F, + 0x6E6E6E6E, + 0x6E6D6C6B, + 0x7271706F, + 0x72747575, + 0x6C6D6E70, + 0x75746F6B, + 0x75737273, + 0x8985807C, + 0x8285888A, + 0x81818181, + 0x80808080, + 0x84838382, + 0x83838484, + 0x82818182, + 0x88888785, + 0x82848686, + 0x797A7D80, + 0x7A7A7978, + 0x797A7A7A, + 0x7E7B7A79, + 0x83828181, + 0x81818181, + 0x80808181, + 0x88868280, + 0x80828688, + 0x87868584, + 0x85868787, + 0x777A7D7E, + 0x6F707274, + 0x827D7672, + 0x7E818385, + 0x7E818181, + 0x686C7279, + 0x7F7D7C7B, + 0x86858381, + 0x82838382, + 0x7C7C7E80, + 0x7D7D7C7C, + 0x78797A7C, + 0x807A7979, + 0x71798183, + 0x817B7570, + 0x80818283, + 0x7E7C7B7B, + 0x85848381, + 0x7D7E7F80, + 0x696E747A, + 0x6A6C6E70, + 0x77746F6C, + 0x6566686B, + 0x706E6B68, + 0x7F7D7773, + 0x6A686E78, + 0x62686D6E, + 0x686A6862, + 0x8A807167, + 0x6872808A, + 0x736E6865, + 0x74767776, + 0x716C7077, + 0x70777D7A, + 0x756C625D, + 0x7F80807D, + 0x6E757C81, + 0x6C6D6D6D, + 0x726E6D6E, + 0x7A7A7A77, + 0x70797873, + 0x6A625E63, + 0x74767879, + 0x7A767372, + 0x68727A80, + 0x66626062, + 0x79787168, + 0x6F747878, + 0x817C7A78, + 0x908E8A86, + 0x727A8790, + 0x8D867A73, + 0x7A7A8087, + 0x6E757C7D, + 0x8D8D8B8A, + 0x898A8B8C, + 0x8D898787, + 0x7E848C8E, + 0x76767677, + 0x77777676, + 0x7C757374, + 0x898C8C86, + 0x6E6E7580, + 0x77747371, + 0x7F797574, + 0x8E8E8B86, + 0x7C808589, + 0x7476797A, + 0x7173777A, + 0x6D6E6F70, + 0x7E766E6A, + 0x85868683, + 0x7B7A797A, + 0x7C7E7F7E, + 0x6E71767A, + 0x6E6D6D6C, + 0x867E7772, + 0x84878989, + 0x777C7F80, + 0x61646A71, + 0x6C706E67, + 0x6C6B6868, + 0x60626465, + 0x5C5C5C5E, + 0x4F4C4D4F, + 0x6A655C55, + 0x64616266, + 0x7575736C, + 0x7877797A, + 0x7A7A7A79, + 0x73747474, + 0x77767474, + 0x6F707374, + 0x67696C6E, + 0x6D6B6A69, + 0x7474716F, + 0x76757474, + 0x79787877, + 0x65686F74, + 0x6F6E6966, + 0x87817B77, + 0x8B939690, + 0x99959291, + 0x92939699, + 0x8C8C9094, + 0x878B8D8D, + 0x7E79797E, + 0x666D7981, + 0x807E756E, + 0x7775767A, + 0x74747575, + 0x6E6F7072, + 0x74706E6E, + 0x87847E79, + 0x85828387, + 0x868A8D8A, + 0x7071767B, + 0x6E717473, + 0x68696B6C, + 0x68686868, + 0x6B67686B, + 0x81858377, + 0x5C5C5E60, + 0x5B5C5D5C, + 0x59565859, + 0x65737668, + 0x5A565657, + 0x63666662, + 0x57555453, + 0x67635F5A, + 0x6F6E6E6D, + 0x74747371, + 0x6F717272, + 0x68686B6E, + 0x68676565, + 0x66656567, + 0x6C6B6762, + 0x5D5F6469, + 0x675C5550, + 0x6E6F716E, + 0x7A756E69, + 0x6F72767A, + 0x81807972, + 0x6B6D727A, + 0x6F646069, + 0x6E727070, + 0x6C68696D, + 0x7B7C7A74, + 0x72737475, + 0x76747472, + 0x77767676, + 0x76767677, + 0x8D83838D, + 0x6F707C8C, + 0x716A6768, + 0x74787B79, + 0x73747677, + 0x6E6E6F71, + 0x62655878, + 0x69565C6B, + 0x1410164B, + 0xE8E2C73D, + 0xD0EBD5EB, + 0xEAEAEBEB, + 0xEBEBD0D8, + 0xD4DCEBD0, + 0x6A7ADEEB, + 0x5762636C, + 0x52706977, + 0x10274110, + 0xE0CBE8EB, + 0xEBE0E6EB, + 0xDBC3DAEB, + 0xE8EBEBC3, + 0x3835C1EB, + 0x6E4E5031, + 0x66686869, + 0x64646465, + 0x8C5C5A75, + 0x8C907670, + 0x311D8B7B, + 0xCBEBC525, + 0x81DDE3EB, + 0x605F687C, + 0x67676767, + 0x67676767, + 0x21155B6F, + 0xEBB0222B, + 0xC0EBE0E3, + 0xEBEBE8E0, + 0xDADBE0E4, + 0xD5EBBFE2, + 0x1DC5DEEB, + 0x59431E25, + 0x61646A6E, + 0x6B686360, + 0x6B686564, + 0x7776746F, + 0x61666F75, + 0x68666361, + 0x5A5F6A75, + 0x736E6E64, + 0x63685363, + 0x101C2D3F, + 0xE9EBE6E7, + 0x6171D6E2, + 0x51525252, + 0x4F505050, + 0x5754443D, + 0x1D371047, + 0xCAD7B939, + 0xE4E6EBE7, + 0xEBEBE2E8, + 0xE4D8EADB, + 0x2EDEEAEB, + 0x32293113, + 0x53504D4B, + 0x57575656, + 0x5B5C5B5B, + 0x54555759, + 0x55555555, + 0x55555555, + 0x51694B5F, + 0x1B371054, + 0xCCEBDC2B, + 0xE3E6DCEB, + 0xE9E9E9E9, + 0xE9E9E9E9, + 0xE0E8EBE4, + 0x6C63E8DC, + 0x65676E75, + 0x535A6265, + 0x19152410, + 0xDBE2EBCA, + 0xE6E9D1EB, + 0xD7E9E8D2, + 0xDAEBEBEB, + 0xDEE2DEE7, + 0x40104ED0, + 0x3B44564F, + 0x4A4B4C4C, + 0x4C4C4B4A, + 0x4A4D5155, + 0x514F4B49, + 0x16412F3C, + 0xE0341F19, + 0xE3E7BEEB, + 0xEBEBE4D0, + 0xDBE2EBE5, + 0xEBE9E0E7, + 0x491356E2, + 0x313F5050, + 0x44566257, + 0x2E13483A, + 0xEBE61910, + 0xE7EBEBD1, + 0xE2EBDFDD, + 0xEBE8EBDC, + 0xE7EBD6EB, + 0x633C1035, + 0x7A746D68, + 0x6F73787A, + 0x6C6E6F70, + 0x49515D67, + 0x55505363, + 0x70727463, + 0x1A191051, + 0xE8D2D03E, + 0xCAE0E4EB, + 0xE2EADDEB, + 0xEBEBDFEB, + 0xD2EBDDD0, + 0x5F7ADCEB, + 0x7A786267, + 0x535C7880, + 0x105A5E50, + 0xEBE6EBE7, + 0xCADBC7EB, + 0xE8E8E8E9, + 0xE3E6E8E9, + 0xCBE8DBE3, + 0x79817DC3, + 0x77726E6E, + 0x61687379, + 0x88837568, + 0x72747A83, + 0x65666B6F, + 0x61646867, + 0xD1104A50, + 0xE8E3E8DE, + 0xE2E2E2E2, + 0xE2E2E2E2, + 0xE5E5E6E6, + 0xE4E4E4E5, + 0x484A5F5E, + 0x293C5040, + 0xD63A101C, + 0xE8EBE3E4, + 0xDEEBEBEA, + 0xDFEBE7D7, + 0xE8EBEBE4, + 0xEAD2EBD5, + 0x40342A10, + 0x6164695F, + 0x62616264, + 0x555B6264, + 0x383D4043, + 0x30303134, + 0x7F797574, + 0x89898885, + 0x85878B8C, + 0x7D7E7F81, + 0x81828282, + 0x7C7D7F81, + 0x7A7A7977, + 0x7B7A7879, + 0x7C7C7D7D, + 0x7D7D7C7C, + 0x87848180, + 0x8E8D8D8A, + 0x77747E89, + 0x81878A82, + 0x8788898A, + 0x7F808285, + 0x6E707476, + 0x7C79746F, + 0x8C837D7A, + 0x94979994, + 0x7C808589, + 0x7F7F7D7C, + 0x7F7D7B7A, + 0x8E8B8581, + 0x8A8E979F, + 0x8D878587, + 0x91929191, + 0x888A8D8F, + 0x7D868B8A, + 0x8E8F887E, + 0x8B8F887B, + 0x88898786, + 0x89878686, + 0x888A8C8C, + 0x81828284, + 0x6E6E737B, + 0x78747372, + 0x7E7F7E7C, + 0x7B79797A, + 0x87878480, + 0x7F83888B, + 0x666B7279, + 0x58585C61, + 0x716D655C, + 0x837D756F, + 0x88878787, + 0x8E909191, + 0x88898B8D, + 0x888D8A86, + 0x7A797980, + 0x8482807F, + 0x89888787, + 0x84838485, + 0x7D808384, + 0x81817F7D, + 0x81817F80, + 0x797A7A7A, + 0x7A7A7979, + 0x77787878, + 0x75767677, + 0x70727577, + 0x6C6E6F70, + 0x8881746B, + 0x81828689, + 0x76767472, + 0x7E7A7675, + 0x83857F85, + 0x7A8E897C, + 0x7579736C, + 0x7E746C6E, + 0x75757576, + 0x7A797876, + 0x7C7C7C7C, + 0x7F7F7E7D, + 0x7C808282, + 0x78767678, + 0x857D797A, + 0x93908E8C, + 0x929B9E9C, + 0x8F888387, + 0x878E9396, + 0x72747A81, + 0x7D777370, + 0xAAA19387, + 0xC1C6CACA, + 0x8E9EB0BA, + 0x857E7977, + 0x7E7C7F84, + 0x9C958A81, + 0x888D959C, + 0xABABA49A, + 0x8F9DA8AB, + 0x74777A7D, + 0x71737474, + 0x6A6C6D6E, + 0x5F616468, + 0x62606062, + 0x62656665, + 0x81746E6C, + 0x9699988E, + 0x858C8F8D, + 0x767F8483, + 0x81817A73, + 0x817D7A7C, + 0x6F757B7E, + 0x978A796E, + 0x686F869A, + 0x817D776E, + 0x858A8983, + 0x7B828583, + 0x857F7C7B, + 0x757D8789, + 0x7A716E70, + 0x827D7B7E, + 0x777B7C7B, + 0x7F7F7C78, + 0x7E818189, + 0x808E9F8D, + 0x88898581, + 0x84818183, + 0x7A7A7B7F, + 0x797B7E7D, + 0x797A7A7A, + 0x7A7A7979, + 0x77787979, + 0x75757676, + 0x74747474, + 0x74747474, + 0x79787777, + 0x7979797A, + 0x81817C7A, + 0x7D7E7F81, + 0x7F818284, + 0x8482817F, + 0x7C7E8185, + 0x8382817E, + 0x7F818385, + 0x7A7A7B7D, + 0x78787979, + 0x7A797878, + 0x76777879, + 0x74747474, + 0x75757575, + 0x76767676, + 0x71727272, + 0x70707171, + 0x6F6E6E6E, + 0x7070706F, + 0x70717272, + 0x696A6C6E, + 0x75746E68, + 0x716F6E72, + 0x88837B76, + 0x7F83878A, + 0x7B7B7B7B, + 0x7F7E7D7B, + 0x82818181, + 0x82828283, + 0x82818182, + 0x87878785, + 0x81848585, + 0x74777A7E, + 0x7B797776, + 0x7F7F7E7D, + 0x7F7E7D7C, + 0x7F7F807F, + 0x7A7B7D7E, + 0x7C7B7A7A, + 0x8B86807B, + 0x8D8E8F8E, + 0x86878787, + 0x81818385, + 0x7074787A, + 0x6A6A6B6E, + 0x807A736E, + 0x7B7E8182, + 0x787A7978, + 0x63676E74, + 0x7E7B7A79, + 0x81818180, + 0x7C7F8080, + 0x74747578, + 0x77767574, + 0x74747576, + 0x6D6B6E70, + 0x74747470, + 0x7A777474, + 0x7F81817F, + 0x77747578, + 0x8081817D, + 0x7A7B7B7C, + 0x696E7479, + 0x6465686A, + 0x746F6A66, + 0x6D6E6F70, + 0x72716F6E, + 0x837E7874, + 0x7A797B81, + 0x68737C81, + 0x68696866, + 0x8F816F62, + 0x6D788792, + 0x6966625F, + 0x67686A6B, + 0x77707074, + 0x7F858781, + 0x6E6C6C6C, + 0x78787673, + 0x6E767C7D, + 0x6E6D6A69, + 0x72716F6E, + 0x7E7B7774, + 0x6D7A7B78, + 0x5E56525B, + 0x6E727577, + 0x807A736E, + 0x68737D83, + 0x63605D60, + 0x7F7C7368, + 0x777F8280, + 0x807D7C7B, + 0x8C8A8783, + 0x7C81898F, + 0x8C88827D, + 0x7A7A7E81, + 0x777B7E7E, + 0x8D8D8D8B, + 0x8787878A, + 0x8A878585, + 0x7980878B, + 0x74737373, + 0x70737576, + 0x7D777575, + 0x898A8884, + 0x6E6E7681, + 0x73717170, + 0x7C777473, + 0x84858581, + 0x74767C81, + 0x6E717373, + 0x71717374, + 0x71737373, + 0x88837C78, + 0x7D818789, + 0x78747271, + 0x8081807C, + 0x686F7981, + 0x6A6A6968, + 0x857C736C, + 0x88888988, + 0x7A7F8182, + 0x5D626A72, + 0x70726E67, + 0x7674706E, + 0x6A6C6D6E, + 0x66666868, + 0x59585A5C, + 0x6A68635D, + 0x57596168, + 0x6867635C, + 0x78746F6C, + 0x6A6E7478, + 0x6B696868, + 0x72706E6D, + 0x6C6E7072, + 0x6567696B, + 0x6B686765, + 0x7473716E, + 0x76747372, + 0x7B7A7978, + 0x6A6E747A, + 0x6F6E6C69, + 0x7A767574, + 0x83888981, + 0x918B8786, + 0x8C8C8E92, + 0x908C8988, + 0x92939493, + 0x7A777A81, + 0x636A747B, + 0x7F7B736C, + 0x7A797A7D, + 0x74757677, + 0x71717273, + 0x74737171, + 0x817F7B78, + 0x807C7E81, + 0x83878A87, + 0x7171757A, + 0x6C707474, + 0x67686869, + 0x6B696868, + 0x736E6E71, + 0x888E8D81, + 0x5F606367, + 0x5F616261, + 0x56565B5A, + 0x606E7263, + 0x59565555, + 0x5E5E5E5C, + 0x5A5A5857, + 0x62605C5B, + 0x68676665, + 0x6D6C6B69, + 0x6E6E6E6E, + 0x6466686B, + 0x62606164, + 0x6A666362, + 0x73726F6C, + 0x66686B6F, + 0x6E68625E, + 0x6D6E7070, + 0x74716D6A, + 0x79797977, + 0x82817A75, + 0x696E777F, + 0x6F666269, + 0x6C706F6E, + 0x68686A6E, + 0x7373716D, + 0x70707171, + 0x76757472, + 0x80818181, + 0x7A7B7D7F, + 0x84807A79, + 0x7B75757F, + 0x74757A7F, + 0x70747675, + 0x6F707172, + 0x6F6F6E6E, + 0x74766F80, + 0x54285C79, + 0xE6C81A56, + 0xE5CBEBE0, + 0x212CBBE8, + 0x101E1010, + 0x1D191F30, + 0xEBEBDE10, + 0x6B78D4DC, + 0x7D7E7472, + 0x646F5C6F, + 0xEAE8C937, + 0xE4EBE4E4, + 0x25101FD6, + 0x23172710, + 0xCFEBBF26, + 0xE0D2EBD6, + 0x936E7AB5, + 0x63656868, + 0x5F5F6062, + 0x60567A55, + 0x7A68625B, + 0xC6117F6A, + 0xEBE4EBB5, + 0x66DDE0DE, + 0x626A5F67, + 0x61616060, + 0x62626261, + 0xBF386560, + 0xE2EBD5E9, + 0x1CDEEBEB, + 0x101A2419, + 0x161C1914, + 0xEBC92225, + 0xEBE9EBE6, + 0x7560D7E4, + 0x6064696D, + 0x69676260, + 0x68626060, + 0x7A78746E, + 0x62646A70, + 0x60626262, + 0x6E6C5F60, + 0x4B645C72, + 0x62716E62, + 0xE8DE3A6D, + 0xE6E8DCDB, + 0x646FE1E5, + 0x4D4C4C4C, + 0x4F4E4E4D, + 0x55544D4F, + 0xDCD63E5F, + 0xEBCCEBDB, + 0x152DC6EB, + 0x101D2512, + 0xDEC52731, + 0xD0EBD7E9, + 0x5772D6E1, + 0x504F4D4B, + 0x55555452, + 0x595B5C5C, + 0x51535557, + 0x53535353, + 0x53535353, + 0x5B724E56, + 0xE3C5316D, + 0xE6C5EBE4, + 0x1927C2EB, + 0x10101010, + 0x10101010, + 0xEBEAD11B, + 0x6A56EBEB, + 0x565C666D, + 0x4A4D5154, + 0xD0E3CA44, + 0xEBC7E8EB, + 0x312525C1, + 0x17212613, + 0x221A1016, + 0xEBE9EBCA, + 0xE8EBDBD0, + 0x5A585674, + 0x494A4A4B, + 0x4D4C4A4A, + 0x4B4E5254, + 0x4F4D4B4A, + 0x2D3B455E, + 0xD6C9EBC6, + 0xEBEBDFC7, + 0x1010111B, + 0x12101010, + 0xB7EBEBEB, + 0xDDE7CCC8, + 0x48474062, + 0x3A495653, + 0x99134062, + 0xE6E5EBE5, + 0x24D9EBDA, + 0x20211816, + 0xCA31171F, + 0xEADCEBDF, + 0x7DC7E2E5, + 0x7A756F6B, + 0x72757A7B, + 0x6C6E7275, + 0x444E5C66, + 0x5245455E, + 0x553B5F6C, + 0xDECA106B, + 0xEBD4EBCB, + 0x1E1FB0D1, + 0x102A1913, + 0x1013101B, + 0xE4EBC410, + 0x6070CED2, + 0x6060616D, + 0x66697D7E, + 0x105D6962, + 0xCDDAEBEB, + 0x103741B3, + 0x15121111, + 0x10131818, + 0x101B1714, + 0x756D561F, + 0x6E6A6D73, + 0x60656D70, + 0x998B6F58, + 0x80828A96, + 0x62656C72, + 0x64666764, + 0xDF236268, + 0xEAE8EBE8, + 0x1F1F1F1F, + 0x1F1F1F1F, + 0x14151515, + 0x13131314, + 0x5257696C, + 0x27496457, + 0xEBE2E3E0, + 0xEBD7D1EB, + 0x25191013, + 0x17101018, + 0xBC151014, + 0xEBE0EBEB, + 0x58B7E5EB, + 0x675D5B54, + 0x615F5F5F, + 0x52585E62, + 0x393F4447, + 0x2C2D3034, + 0x817A7674, + 0x87898A87, + 0x82848687, + 0x80808081, + 0x7F818181, + 0x797A7B7D, + 0x76777776, + 0x77757474, + 0x81818283, + 0x83828181, + 0x89878584, + 0x8D8D8D8B, + 0x7A7A838D, + 0x81878983, + 0x88898988, + 0x7D7F8387, + 0x696D7275, + 0x76736E69, + 0x877F7A7A, + 0x8D919390, + 0x7B7D8186, + 0x8786837F, + 0x7A7E7F7F, + 0x7F7C7978, + 0x807E848C, + 0x89878583, + 0x918F8C89, + 0x898B8D90, + 0x83868A8D, + 0x97928B86, + 0x81828689, + 0x88888783, + 0x8D898786, + 0x898C8F8F, + 0x7577787A, + 0x6664686F, + 0x7A757270, + 0x81818280, + 0x7776787A, + 0x7E7D7C79, + 0x797B7F81, + 0x65696F74, + 0x56575C61, + 0x706B635B, + 0x7C7A7876, + 0x8D89837F, + 0x888A8A8B, + 0x81838587, + 0x8C8D867E, + 0x7E7C7E84, + 0x817E7C7A, + 0x87868583, + 0x84858586, + 0x797B8083, + 0x74767778, + 0x77747272, + 0x7D7A7877, + 0x81818180, + 0x7A7B7C7D, + 0x74757678, + 0x6C6E7376, + 0x6667686A, + 0x8C867669, + 0x81808288, + 0x77797877, + 0x7F7A7675, + 0x8C8E858A, + 0x7A91897D, + 0x797B756E, + 0x82797172, + 0x7A7A7978, + 0x78797A7A, + 0x73747679, + 0x83807A75, + 0x7C818587, + 0x7A777677, + 0x8D817A79, + 0x93939595, + 0xA9A79E95, + 0x99999EA5, + 0x8A8D8F90, + 0x81838587, + 0x7C7A7B7C, + 0x9F988C81, + 0xAEB7B1A8, + 0x8287929F, + 0x827E7A7A, + 0x83818182, + 0x958F8781, + 0x8E909396, + 0xA8AAA59B, + 0x8D9AA5A6, + 0x71798187, + 0x6365686C, + 0x595C5F61, + 0x52525456, + 0x59575656, + 0x605F5D5B, + 0x78706862, + 0x82817E7C, + 0x858E8F89, + 0x757E8181, + 0x82817C77, + 0x82807E80, + 0x757A7E80, + 0x9F928176, + 0x7275879A, + 0x85827E78, + 0x7B848783, + 0x747B7D7A, + 0x827A7473, + 0x6E798588, + 0x72686468, + 0x7D767476, + 0x7677787A, + 0x867C7474, + 0x878D878B, + 0x7E828F89, + 0x83878582, + 0x807D7C7F, + 0x7E7E7D7B, + 0x7C7E7F7E, + 0x797A7C7D, + 0x76777778, + 0x76767777, + 0x74747475, + 0x79787675, + 0x75767879, + 0x7A797878, + 0x7A7A7A7A, + 0x8685817E, + 0x81828385, + 0x86858584, + 0x84858586, + 0x80818385, + 0x83838381, + 0x7A7E8184, + 0x78787778, + 0x76767677, + 0x7A7A7876, + 0x75767777, + 0x75747474, + 0x75747474, + 0x76767675, + 0x74747575, + 0x74747474, + 0x73747474, + 0x72727273, + 0x74747473, + 0x70717374, + 0x7476716C, + 0x706D6C6F, + 0x87807772, + 0x7A7F8588, + 0x78777878, + 0x7D7C7A79, + 0x8181807F, + 0x81818182, + 0x81808181, + 0x84858482, + 0x81838485, + 0x7174787D, + 0x77747270, + 0x83817F7B, + 0x81818181, + 0x7D7E7F80, + 0x787A7E81, + 0x7F7C7A78, + 0x8F8B8682, + 0x8D8F9191, + 0x82858789, + 0x7B7B7D80, + 0x6B6F7476, + 0x66666768, + 0x7D776F6A, + 0x797B8081, + 0x70737475, + 0x5F62676B, + 0x7D7A7875, + 0x7E7F807F, + 0x757B7E7F, + 0x6868696E, + 0x6F6E6C6A, + 0x6F6F7070, + 0x62626669, + 0x7D776E66, + 0x73747A81, + 0x7E7F7D78, + 0x706D7074, + 0x797C7E79, + 0x78787878, + 0x686C7276, + 0x61626568, + 0x736E6862, + 0x76777676, + 0x75747475, + 0x88817A79, + 0x8987898B, + 0x757D8587, + 0x75777673, + 0x83807A76, + 0x74787E83, + 0x6B6A6866, + 0x6365686A, + 0x766E6C6C, + 0x83868781, + 0x73757C81, + 0x7C7A7874, + 0x6F7A7E7C, + 0x72706C69, + 0x6F707070, + 0x75747270, + 0x6771706B, + 0x5B534E56, + 0x74747474, + 0x7C797674, + 0x68737B80, + 0x5F5D5C60, + 0x817F7468, + 0x838A8A84, + 0x7F7E7F7F, + 0x84838180, + 0x83848688, + 0x81828383, + 0x7D7A7878, + 0x88878681, + 0x8B8D8C8A, + 0x82828387, + 0x89878585, + 0x777D8689, + 0x7D777474, + 0x72798081, + 0x82807E7D, + 0x8D8B8986, + 0x73747E89, + 0x72717274, + 0x77737170, + 0x7A7B7C7A, + 0x6B6D7378, + 0x686B6D6C, + 0x736E6B6A, + 0x7F7F7D79, + 0x8B87827F, + 0x7C81868A, + 0x81807F7F, + 0x83858584, + 0x6A707B84, + 0x6C6D6D6A, + 0x837D746D, + 0x81818284, + 0x7A7F8181, + 0x5E626B74, + 0x74757068, + 0x817E7974, + 0x76767675, + 0x75757676, + 0x6D6D7073, + 0x7272716E, + 0x585C666E, + 0x595C5C5B, + 0x7C74685E, + 0x6068737B, + 0x615E5C5C, + 0x6B696764, + 0x68696C6E, + 0x62636667, + 0x68646260, + 0x72706E6B, + 0x75747271, + 0x7D7C7A78, + 0x70747A80, + 0x71706F6E, + 0x6F6D6C6B, + 0x747A7C76, + 0x81787372, + 0x817F8083, + 0x8382807C, + 0x82818182, + 0x7573767D, + 0x62687177, + 0x7E786F69, + 0x7D7D7F80, + 0x74767879, + 0x71727274, + 0x74737271, + 0x7A797876, + 0x7A76777A, + 0x7F838581, + 0x7171757A, + 0x6A6E7374, + 0x68686868, + 0x6E6D6A68, + 0x74747473, + 0x7F83827B, + 0x696A6E71, + 0x62676A6B, + 0x565A5D5B, + 0x5F6D7062, + 0x59595755, + 0x56565657, + 0x5B5C5C5A, + 0x5A595758, + 0x5E5C5C5B, + 0x6262615F, + 0x6B6C6C6C, + 0x61636669, + 0x5D5B5D63, + 0x6E686260, + 0x7A7A7A78, + 0x6E6F7276, + 0x7B7D7A77, + 0x6C6D7076, + 0x6E6C6D6E, + 0x827F7A73, + 0x837F7A76, + 0x6B737C83, + 0x6F686268, + 0x6E73706F, + 0x6B6B6C6D, + 0x6E6E6D6C, + 0x706E6E6E, + 0x76757472, + 0x7D7F7F80, + 0x7475787A, + 0x76787168, + 0x7E746C6E, + 0x7A828788, + 0x65666870, + 0x6B6B6C6C, + 0x6E6E6D6B, + 0x6C6C7470, + 0x56105A76, + 0xE2EB211E, + 0xDCE4DCDC, + 0x6653E3EB, + 0x62745042, + 0x57597F52, + 0xE9DBDB2E, + 0x1E3FC7EB, + 0x7E755332, + 0x667D7290, + 0xE8E8CE29, + 0xD0DAEBE1, + 0x876E67EB, + 0x6B6C7E62, + 0xC7EBCC43, + 0xE6DAEAEB, + 0x60688BEB, + 0x62636668, + 0x5C5C5D5F, + 0x15196231, + 0x13101E23, + 0xEB101C19, + 0xEBC5EBE7, + 0x5FEBEBE9, + 0x5766565E, + 0x5D5D5D5C, + 0x5F5F5E5E, + 0xCF13635C, + 0xD1E5E9E7, + 0x41E2EBE5, + 0x44716E39, + 0x565C5044, + 0xEBD72C54, + 0xEBC1E2E8, + 0x8656EBE2, + 0x5D616669, + 0x65625F5C, + 0x62606162, + 0x6D6E6C68, + 0x5D5E6368, + 0x55595C5D, + 0x5756574C, + 0x1F4C1047, + 0x10101710, + 0xE1DA103E, + 0xE3E8E7EB, + 0x676FE9E7, + 0x4B4A4949, + 0x50504F4D, + 0x535A5150, + 0xDEDD2556, + 0xE7E1EBEB, + 0x657EE2E8, + 0x2B4A6350, + 0xE8E12852, + 0xE7E6EBE2, + 0x4B40D1EB, + 0x4A494949, + 0x5C59534E, + 0x575A5C5C, + 0x51525355, + 0x51515151, + 0x52525251, + 0x1E104060, + 0xEBBD261A, + 0xE1EBE6E8, + 0x2E74EBD6, + 0x53535353, + 0x53535353, + 0xEBE0D51D, + 0x1F2FC0DC, + 0x55514A42, + 0x4B4C4F53, + 0xDCDED424, + 0xD9ABEBE3, + 0x453755EB, + 0x382F3B38, + 0x404F3943, + 0xE8E2D0CF, + 0xCAE8EAEB, + 0x48595153, + 0x4949494A, + 0x52504D4A, + 0x4F505152, + 0x4D4D4E4E, + 0x19475D5C, + 0xE9EBE4CA, + 0xEBEAE4EB, + 0x5E5F6867, + 0x1D33344C, + 0xEBDCE9E9, + 0xD1EBE8EB, + 0x545F4B4E, + 0x4A42464A, + 0xE920436E, + 0xEBC3EBE9, + 0x4FE9EBE2, + 0x4444424E, + 0xDC423C45, + 0xD7EBEBD7, + 0xA2EBEBDA, + 0x706B6662, + 0x6C6E7172, + 0x696D6E6E, + 0x444C5762, + 0x51424258, + 0x6A1A4971, + 0xEBEB141C, + 0xD0EBDDD0, + 0x756EE4EB, + 0x565E6354, + 0x4C5F5C39, + 0xE9DCE148, + 0x203CCFEB, + 0x746A5F49, + 0x83777A70, + 0x30849287, + 0xEBE7EBEB, + 0x3D6866E5, + 0x625B5655, + 0x61666A68, + 0x5E646D50, + 0x4E736666, + 0x6A69737F, + 0x69696C6D, + 0x92866F5C, + 0x81828790, + 0x62656C72, + 0x61626362, + 0xD4104953, + 0xE9E0E6E1, + 0x61616161, + 0x61616161, + 0x72727171, + 0x73737372, + 0x60696F71, + 0x25557271, + 0xE9EBD8DF, + 0xEBEBE9DE, + 0x4A605E50, + 0x6A6D563F, + 0xCC305161, + 0xDEE9CAE3, + 0x5CDAEBEA, + 0x5C596762, + 0x5F5C5A58, + 0x55585C5F, + 0x424A5156, + 0x2B2D3139, + 0x827A7572, + 0x87898A88, + 0x81818181, + 0x82828181, + 0x7D7E7F80, + 0x7576787A, + 0x74757676, + 0x74737172, + 0x85868687, + 0x87868685, + 0x8A888786, + 0x8D8D8D8C, + 0x7F81888E, + 0x83848481, + 0x8A8B8A89, + 0x7A7D8287, + 0x6A6E7479, + 0x77736E69, + 0x857E7A7A, + 0x868B8E8C, + 0x7E7D8185, + 0x8E8D8983, + 0x7A7F8181, + 0x72707074, + 0x7871737A, + 0x86868581, + 0x8E8B8682, + 0x8C8D8D8F, + 0x8583878D, + 0x938D8787, + 0x77778799, + 0x8585847F, + 0x908C8887, + 0x8A8D9193, + 0x6D6F7174, + 0x5F5D6067, + 0x7C76716E, + 0x81838482, + 0x73747679, + 0x76767574, + 0x74757778, + 0x64686E71, + 0x56575C62, + 0x706B635B, + 0x72757A7F, + 0x938A7E74, + 0x81838484, + 0x7A7C7E80, + 0x8D8C8279, + 0x807F8187, + 0x7E7C7A79, + 0x82828181, + 0x84868686, + 0x74777C81, + 0x6A6C6E6E, + 0x78746E6B, + 0x817B7774, + 0x88878784, + 0x7C7E8081, + 0x7476777A, + 0x6A6E7275, + 0x67686969, + 0x90908376, + 0x72717784, + 0x7A7F8181, + 0x817D7878, + 0x92968889, + 0x7990877D, + 0x7A7C7973, + 0x7A747073, + 0x88888786, + 0x7D7F8387, + 0x6D6E7375, + 0x87817870, + 0x7C828789, + 0x7B797677, + 0x94847A78, + 0x93959A9D, + 0xB2A9998E, + 0x9BA0AAB1, + 0x8C898887, + 0x92918F8D, + 0x7C818689, + 0x938D837D, + 0xA5B0A594, + 0x837F808E, + 0x83838485, + 0x87838182, + 0x908B8580, + 0x93939392, + 0x989D9992, + 0x818D9797, + 0x6A747E85, + 0x60616265, + 0x575C5F62, + 0x52525354, + 0x55555453, + 0x58565554, + 0x6868615A, + 0x786E6464, + 0x85918F84, + 0x787E7E7C, + 0x7F7C7A78, + 0x83828180, + 0x7A7E807F, + 0x998D7F78, + 0x817B818C, + 0x87878787, + 0x737E8381, + 0x70767771, + 0x877D7775, + 0x727E8B8E, + 0x6E646266, + 0x746E6C70, + 0x706D6B6E, + 0x87776C6E, + 0x919B8E8A, + 0x867D8386, + 0x777B7B7A, + 0x74727073, + 0x83827F7B, + 0x7F818383, + 0x797B7D7E, + 0x74747577, + 0x75767676, + 0x74747474, + 0x7B7A7776, + 0x76777A7B, + 0x7A7A7978, + 0x7A7A7A7A, + 0x89878481, + 0x85858687, + 0x8A888684, + 0x8486888A, + 0x82828385, + 0x83848483, + 0x777B8083, + 0x76757474, + 0x75747575, + 0x7A7A7876, + 0x74757676, + 0x75757474, + 0x75747474, + 0x76767675, + 0x77777778, + 0x76767676, + 0x76777878, + 0x74747475, + 0x7A797776, + 0x7A7A7A7A, + 0x74787572, + 0x706C6A6E, + 0x867E746E, + 0x777C8387, + 0x76767677, + 0x7C7B7A77, + 0x81807F7F, + 0x81818181, + 0x7F7E8081, + 0x81828281, + 0x81838484, + 0x6F73777C, + 0x736E6B69, + 0x85827D78, + 0x82828383, + 0x7D7E7F81, + 0x7A7D8184, + 0x82807B7A, + 0x93928F8D, + 0x86898D92, + 0x7F838789, + 0x7777797B, + 0x686D7274, + 0x64636365, + 0x7C756D68, + 0x787A7E7F, + 0x6A707578, + 0x5C5C6064, + 0x7D7A7674, + 0x7C7D7F7F, + 0x71797D7F, + 0x5F5F6168, + 0x6A686563, + 0x6C6C6C6C, + 0x68676868, + 0x8A82766D, + 0x6E73818D, + 0x7C7C7A73, + 0x6C686C73, + 0x747A7C76, + 0x6F6F6F6F, + 0x60646A6E, + 0x60616467, + 0x746F6862, + 0x7D7D7B7A, + 0x78787A7B, + 0x8B827D7D, + 0x92919291, + 0x7E828687, + 0x8285837F, + 0x757D878D, + 0x78757271, + 0x7A7A7978, + 0x7174767A, + 0x79716E6E, + 0x84868681, + 0x797F8890, + 0x84827E7A, + 0x747F827F, + 0x78756F6D, + 0x6E6E7073, + 0x6A6C6E6E, + 0x6E757068, + 0x69615B62, + 0x7C7A7571, + 0x7374787B, + 0x6971787A, + 0x5C5C5C62, + 0x7A797064, + 0x878C897F, + 0x7D7F8181, + 0x7E7E7D7D, + 0x86848281, + 0x777B8185, + 0x7B797775, + 0x8683807D, + 0x898C8B89, + 0x7F7F8184, + 0x89878686, + 0x767D868A, + 0x877F7A79, + 0x78828D8E, + 0x83828181, + 0x8D898684, + 0x7A7D8793, + 0x74747679, + 0x726F6E6E, + 0x74767675, + 0x65676D73, + 0x65686868, + 0x7A6F6866, + 0x92918D84, + 0x8D898582, + 0x86888B8D, + 0x90929395, + 0x86888C8E, + 0x6E727C85, + 0x70737370, + 0x83817972, + 0x78787A80, + 0x7B7F8080, + 0x60646C74, + 0x78787169, + 0x87847E79, + 0x7F7D7B7A, + 0x81818181, + 0x7A7C8083, + 0x787A7B7B, + 0x68696F75, + 0x565D6568, + 0x80746256, + 0x5B657480, + 0x59565351, + 0x6563615C, + 0x5E606366, + 0x5A5B5D5E, + 0x65615D5C, + 0x706E6C69, + 0x75737170, + 0x7E7D7B79, + 0x74797F83, + 0x72727373, + 0x746E6A66, + 0x6B747C7A, + 0x746A6464, + 0x78757578, + 0x7B828381, + 0x6D6C6D73, + 0x726E7279, + 0x62676F74, + 0x7D756E68, + 0x80818281, + 0x7577797A, + 0x71717273, + 0x74737170, + 0x74757574, + 0x77737476, + 0x7B80817E, + 0x71717579, + 0x696E7374, + 0x68686869, + 0x716F6D6A, + 0x787B7A75, + 0x7A7A7976, + 0x7474777A, + 0x666C7375, + 0x595D605C, + 0x626F7162, + 0x595C5855, + 0x54515155, + 0x595C5B59, + 0x52515155, + 0x57565554, + 0x5C5B5A58, + 0x6A6B6B6A, + 0x60626568, + 0x5B575C63, + 0x7168625F, + 0x7F818181, + 0x7374767A, + 0x8184817D, + 0x70707279, + 0x69696D71, + 0x8882796F, + 0x827C7876, + 0x6E768185, + 0x71696268, + 0x70757370, + 0x6F6E6D6A, + 0x706F6E6F, + 0x706E6D6C, + 0x76757472, + 0x71737474, + 0x65686A6E, + 0x6D746F63, + 0x7A706562, + 0x808A8781, + 0x59585E6E, + 0x68686868, + 0x6E6C6A68, + 0x62627E76, + 0xB0377E7F, + 0xE8EBD6E6, + 0x201E19E8, + 0x3F661012, + 0x59484F5C, + 0x81639143, + 0xBB31151E, + 0xD2CAEBD6, + 0x6280A2CA, + 0x747D6177, + 0xE3EBE53A, + 0x1035E2DB, + 0x58605839, + 0x744E617C, + 0x17101C10, + 0xE5EBE2E3, + 0x61616EEB, + 0x60626566, + 0x5A5B5C5D, + 0xBE43496E, + 0xE7EBDED3, + 0xEBE5DBE4, + 0xE2EBE9D2, + 0x68E8E0DF, + 0x535B4C69, + 0x5C5B5B5B, + 0x5C5C5C5C, + 0xEB196A46, + 0xE0E0EBE2, + 0x4D301014, + 0x555F6659, + 0x8D765950, + 0x10421086, + 0xDFE4EBE5, + 0x8563EAE1, + 0x595C6265, + 0x615E5B58, + 0x5E5E6265, + 0x5E606160, + 0x595A5E62, + 0x4E53585A, + 0x423C5C4F, + 0xC3D82844, + 0xEBE5CBEB, + 0xEBDFE2DF, + 0xE9E7E0E3, + 0x3D52E5EB, + 0x4C4A4948, + 0x5251504E, + 0x56625754, + 0xE8E91F56, + 0x28EAE5D6, + 0x3D572110, + 0x626E5253, + 0x10103336, + 0xE2E9D821, + 0x6D39DCE9, + 0x44424344, + 0x6660564B, + 0x56595C5D, + 0x52525254, + 0x50505050, + 0x51515150, + 0xDA354533, + 0xCFEAEBDD, + 0x1C241FD9, + 0x7D381020, + 0x56565656, + 0x56565656, + 0xDB391912, + 0xEBE7EBE8, + 0x4C6891B0, + 0x44474644, + 0xEBE6E61A, + 0x192BD6DB, + 0x49614831, + 0x3D515B41, + 0x103C473A, + 0xEBBF1E2D, + 0xE8E7DCEB, + 0x555C5A6F, + 0x4A4A4949, + 0x5756514E, + 0x52525150, + 0x4C4E5051, + 0x25384446, + 0xE5EBD5EB, + 0x101528D2, + 0x62566E5E, + 0x11565371, + 0xEB101716, + 0xDFE8DAEA, + 0x39372B4C, + 0x5856523D, + 0xCA21473C, + 0xDBE7DCEB, + 0x4124102E, + 0x3E645B5B, + 0x13145431, + 0xEBC31029, + 0x52D6EBE4, + 0x645F5A56, + 0x65666868, + 0x62676562, + 0x3E434B57, + 0x594C4F62, + 0xA6254A7B, + 0xE5E7E7D2, + 0x351932EB, + 0x52681610, + 0x625C607A, + 0x58687246, + 0xC2181716, + 0xD6D2EBBB, + 0xA08FA1CB, + 0x8D818275, + 0x1E798F8D, + 0xE9DCE4E8, + 0x5A7E69E9, + 0x6358524F, + 0x686D706D, + 0x67628C6D, + 0x648E4A75, + 0x6B6D7A8A, + 0x74706E6D, + 0x8481786F, + 0x74757980, + 0x5C60666A, + 0x57595B5C, + 0xE8206074, + 0xEBDDE7EB, + 0x5C5C5C5C, + 0x5C5C5C5C, + 0x6E6D6C6C, + 0x7171706E, + 0x64726E6D, + 0x1B52727A, + 0xE6E4EBE2, + 0x182BC8EB, + 0x554C4245, + 0x6E5D4A4C, + 0x21324644, + 0xE3D37515, + 0x4CE5EBEB, + 0x55516056, + 0x5D5B5653, + 0x595A5B5C, + 0x49545D62, + 0x2B2E353E, + 0x877B7574, + 0x828A9290, + 0x81818181, + 0x84838281, + 0x7A7C7D7E, + 0x77777879, + 0x74747576, + 0x78777574, + 0x87888989, + 0x81808183, + 0x8C87817B, + 0x8A8B8C8D, + 0x84858787, + 0x89878684, + 0x878B8D8D, + 0x7C7E8184, + 0x6B717A82, + 0x756E6868, + 0x817C7774, + 0x82838584, + 0x82818284, + 0x8D8D8A87, + 0x6F757D83, + 0x6E6E6E6D, + 0x7F7C7B7A, + 0x88878481, + 0x8E8B8784, + 0x898C8E90, + 0x85868788, + 0x918E8A87, + 0x767A848C, + 0x87847F79, + 0x8F87878A, + 0x969B9E99, + 0x6D737A7F, + 0x68676768, + 0x7978746E, + 0x74797B7A, + 0x6C6B6E6F, + 0x7A79746F, + 0x686E767C, + 0x68686666, + 0x6D6C6660, + 0x7A746D6B, + 0x6E7B7F79, + 0x817B7068, + 0x7A7B8185, + 0x7A7B7C7B, + 0x817D7A7A, + 0x7A7F8485, + 0x7C787575, + 0x757A7E7F, + 0x7D7D7F80, + 0x797A7D7E, + 0x6A6C6F72, + 0x7B776F6B, + 0x7E7A7776, + 0x87878783, + 0x7B7F8386, + 0x7A7A7979, + 0x6F747576, + 0x6766676A, + 0x8F908A7F, + 0x71697387, + 0x85838180, + 0x81828485, + 0x88898B8D, + 0x81848788, + 0x747A7A77, + 0x79716B6C, + 0x99999894, + 0x8F969A9A, + 0x746C6D7D, + 0x8F877A74, + 0x7C848889, + 0x7B767374, + 0x93908880, + 0x81889193, + 0x91878382, + 0xA5A6A59C, + 0x96959493, + 0x90929395, + 0x7E818890, + 0x8F87817E, + 0x989F9F9B, + 0x8886868D, + 0x87888D91, + 0x85878887, + 0x83848687, + 0x908D8985, + 0x8785878A, + 0x7E858B8B, + 0x63697176, + 0x5C5D5F61, + 0x595C5D5E, + 0x53545657, + 0x52535353, + 0x52525252, + 0x50515353, + 0x6B625750, + 0x817F756D, + 0x7D7A797C, + 0x817B7A7F, + 0x85818184, + 0x807D7C7E, + 0x89848181, + 0x86878681, + 0x81888B88, + 0x73747A80, + 0x8387877C, + 0x8E817A7D, + 0x8C8F9598, + 0x827E7D88, + 0x7174747A, + 0x6B615C62, + 0x7E6F686E, + 0x8B8D8A83, + 0x767F8687, + 0x7B777271, + 0x66646A75, + 0x9A8E7A74, + 0x8F918F94, + 0x75797E83, + 0x6F727474, + 0x76767474, + 0x7B7A7776, + 0x74787B7D, + 0x7A767372, + 0x7A797776, + 0x7A7A7B7B, + 0x88878481, + 0x86868687, + 0x88878584, + 0x84858788, + 0x84838383, + 0x85858584, + 0x747A8083, + 0x71707071, + 0x716F7174, + 0x77797975, + 0x73737577, + 0x79797875, + 0x72747678, + 0x78767472, + 0x77777778, + 0x7A797878, + 0x7E7E7A78, + 0x78787A7C, + 0x7A787675, + 0x7A7A7A7A, + 0x787A7674, + 0x6B6B6E73, + 0x867C7471, + 0x747C878B, + 0x7173777A, + 0x7C797471, + 0x7F7C7A79, + 0x81818181, + 0x7A7C7F81, + 0x817F7C7A, + 0x81808080, + 0x71767D81, + 0x736E6967, + 0x81807C78, + 0x81828383, + 0x82828181, + 0x747B8184, + 0x80797270, + 0x93908B88, + 0x888C9093, + 0x8286898C, + 0x72757A7F, + 0x686B6E71, + 0x60616264, + 0x706B6865, + 0x7C7A7874, + 0x5F6E797F, + 0x57535054, + 0x807B7169, + 0x7C7B7C7F, + 0x737A7B81, + 0x5F5D585F, + 0x68676360, + 0x6C6B6969, + 0x81756F70, + 0x83818386, + 0x6A6E7A84, + 0x7D7B776F, + 0x6C686D72, + 0x69707573, + 0x68746D61, + 0x64564B54, + 0x60616262, + 0x75706862, + 0x7C7A7878, + 0x74787B7E, + 0x87838589, + 0x878D908D, + 0x81818283, + 0x87868482, + 0x72798185, + 0x7D78726F, + 0x8787847D, + 0x77808787, + 0x79727072, + 0x80838581, + 0x777D8386, + 0x87807774, + 0x7C818587, + 0x7F7B7979, + 0x68686F79, + 0x706E6D6B, + 0x8A937E64, + 0x8E776671, + 0x80838586, + 0x7475787C, + 0x6970767A, + 0x5C5D5F63, + 0x747A7263, + 0x787F7D74, + 0x7F81807A, + 0x81848480, + 0x87848281, + 0x7E818587, + 0x80848381, + 0x85797075, + 0x8C8D8C8C, + 0x87888A8B, + 0x87878787, + 0x87878787, + 0x8D808085, + 0x74899D9E, + 0x7E776C62, + 0x85828181, + 0x818A8C8A, + 0x6E6E7076, + 0x73716E6C, + 0x7A787574, + 0x68696F77, + 0x62626468, + 0x746C6662, + 0x9591887F, + 0x8E8C867F, + 0x8A88888B, + 0x93939494, + 0x85878C8F, + 0x74757E89, + 0x7C7A7978, + 0x8A827B7A, + 0x73757E89, + 0x7D7F8080, + 0x676B7178, + 0x6E747471, + 0x7676736D, + 0x827C7875, + 0x898A8A87, +}; + +static gctUINT32 ppuMem5[] = +{ + 0x7E83898D, + 0x807E7B7B, + 0x84817F7F, + 0x5C687A83, + 0x7E766C66, + 0x616A777F, + 0x5C5B5856, + 0x625F5C5C, + 0x52515A62, + 0x52585C58, + 0x60585554, + 0x70716F69, + 0x71696A6E, + 0x787F827D, + 0x72747D87, + 0x6D6D6E71, + 0x877F736A, + 0x6A737F87, + 0x6B686868, + 0x7A78746E, + 0x767A7B7C, + 0x67696E72, + 0x716E6E6E, + 0x686E7274, + 0x7A75706C, + 0x7D7D7D7C, + 0x7477797A, + 0x6E6E6E70, + 0x72716F6E, + 0x74747473, + 0x75747474, + 0x75757575, + 0x72747475, + 0x70707071, + 0x6E6A6868, + 0x76767572, + 0x78757678, + 0x7D7F7F7C, + 0x81807D7A, + 0x686E787F, + 0x625E6062, + 0x67696B68, + 0x62625C55, + 0x5352555C, + 0x5C5C5B59, + 0x5052565A, + 0x56504F50, + 0x68625E5C, + 0x7B7E7468, + 0x6260626E, + 0x59565B62, + 0x726B645E, + 0x7D818283, + 0x76757679, + 0x72747472, + 0x726F6E6E, + 0x73727170, + 0x87817A75, + 0x7C74757A, + 0x747F8786, + 0x706A686E, + 0x747B7872, + 0x78747475, + 0x6F747A7B, + 0x706D6C6C, + 0x72737473, + 0x74797874, + 0x66626269, + 0x6A6A6B6B, + 0x6D6C6A6A, + 0x807B7168, + 0x5C636F7A, + 0x62636566, + 0x65646362, + 0x61516277, + 0xE0226E5C, + 0xE5EAE6E2, + 0x48627ACA, + 0x4C514837, + 0x4E4E4A46, + 0x78575350, + 0xE610687C, + 0xDFE2DBE0, + 0x504E58EB, + 0x1026109F, + 0xEBE6BD38, + 0x7483AEEB, + 0x5E5A5561, + 0x716D645C, + 0x1834566C, + 0xE8EBE2E6, + 0x181531D3, + 0x5B5D6162, + 0x605D5C5A, + 0xD1154469, + 0xEBE8E8EB, + 0xCDEBDCEB, + 0xEBDDEBDC, + 0x5DE6E8EB, + 0x47594A56, + 0x5C5C5550, + 0x5D71564A, + 0xEB10616A, + 0xE3EBD5EB, + 0x3E524456, + 0x64645C56, + 0x8C716370, + 0x2E5C4E55, + 0xE7E8E9EB, + 0x6788B7D9, + 0x50535B60, + 0x605C5651, + 0x5C5D5E5F, + 0x52545659, + 0x57565554, + 0x4B4E5256, + 0x4A705252, + 0xEBD8173F, + 0xCDDFEBD4, + 0xE5EBDCEB, + 0xE8EBE2EB, + 0x4A57E5EB, + 0x4E4C4B4B, + 0x5251504F, + 0x3B105F60, + 0xEBC93914, + 0x5DDFCBEB, + 0x4B58522C, + 0x6259504A, + 0x314A6369, + 0xEBCECA29, + 0x1043E4D6, + 0x4F483E5C, + 0x685E5D3A, + 0x565B5C5C, + 0x4E4D4D50, + 0x50504E4E, + 0x4D4E5050, + 0xDC193F49, + 0xEBE3EAE2, + 0x6F7457C9, + 0x50465B4B, + 0x5860686E, + 0x53535254, + 0xE715402F, + 0xE0EBEBE9, + 0x385662DF, + 0x29403755, + 0xDDEBEB16, + 0x5F3FE7EB, + 0x553D4245, + 0x2A4D4853, + 0x446F233E, + 0xD9C93E3B, + 0xC5EBE5E8, + 0x434F4C71, + 0x4547494A, + 0x5D564E47, + 0x504F5256, + 0x4E515554, + 0x56105012, + 0xC4EBE8DB, + 0x314B87BB, + 0x565A523D, + 0x524A5648, + 0x8725235A, + 0xCBEBE6EB, + 0x23331A53, + 0x5B51394C, + 0xE8103F5C, + 0xD9E2EBE5, + 0x504C293C, + 0x4A544B40, + 0x513E3D40, + 0xE9E11948, + 0x5CE9DEEB, + 0x615D5957, + 0x67676664, + 0x4D84534D, + 0x38584348, + 0x5E3E595A, + 0xDE16627A, + 0xE1EBD4E9, + 0x52607CD6, + 0x5D63686C, + 0x5A585758, + 0x6D5F6552, + 0xE619767C, + 0xEBEBE6E3, + 0x7E626FDE, + 0x8E938083, + 0x31315410, + 0xE8EAEBE8, + 0x84875FD3, + 0x686C7174, + 0x6F6D6967, + 0x5C5A5E5E, + 0x60636A68, + 0x76666462, + 0x74876868, + 0x857A627F, + 0x836B5C93, + 0x695F6462, + 0x124D515B, + 0xDB28121D, + 0xE4E3E9E6, + 0x4A4A4A4A, + 0x4A4A4A4A, + 0x8B626B68, + 0x5458807A, + 0x7878736F, + 0x145D526A, + 0xD8EBEAB7, + 0x648DCADE, + 0x4A544A61, + 0x5D56474F, + 0x3B476A43, + 0xE8EB123C, + 0x4DE2EBE6, + 0x55536059, + 0x55555454, + 0x59585756, + 0x45555C5C, + 0x3B322D34, + 0x8078726F, + 0x81858786, + 0x7F808080, + 0x8181807F, + 0x7C7D7E7E, + 0x7C7C7C7C, + 0x7A7A7A7B, + 0x7D7C7A7A, + 0x83878787, + 0x7A7A7B7F, + 0x8B867E79, + 0x8C8C8D8D, + 0x84868789, + 0x87868484, + 0x86878989, + 0x7A7B7F82, + 0x696D767F, + 0x756E6968, + 0x7A777574, + 0x80807E7C, + 0x817F8081, + 0x8B8A8884, + 0x7A7E8286, + 0x71737578, + 0x807D7B7A, + 0x88878582, + 0x8F8D8887, + 0x888B8D8F, + 0x87878889, + 0x8E8C8987, + 0x74777E85, + 0x89878179, + 0x938D8B8B, + 0x9A9B9B99, + 0x787E858A, + 0x74747474, + 0x6F737472, + 0x6D6F706E, + 0x67656668, + 0x74736F6A, + 0x686A6E72, + 0x65666767, + 0x74736D66, + 0x817A7474, + 0x707A7E7C, + 0x7B746C69, + 0x73737475, + 0x74747474, + 0x7B787676, + 0x74797E7F, + 0x75727171, + 0x73757878, + 0x76747474, + 0x78797A79, + 0x6F747574, + 0x7C756E6C, + 0x7B787675, + 0x8181817F, + 0x7A7C7E80, + 0x79787879, + 0x73777A7A, + 0x69696A6E, + 0x93948D80, + 0x776E778B, + 0x85878582, + 0x86838182, + 0x85858687, + 0x80828485, + 0x70777A7A, + 0x756F696A, + 0xA3A29F9B, + 0x989EA4A4, + 0x77717382, + 0x90897D77, + 0x7E858B8D, + 0x817E7A79, + 0x94948F88, + 0x747E8890, + 0x7A737274, + 0x9C999387, + 0x90929496, + 0x95939290, + 0x81848C93, + 0x8B857F7F, + 0x8D91918E, + 0x89858285, + 0x8B8D9093, + 0x8B8C8D8C, + 0x81828384, + 0x86858382, + 0x85818184, + 0x79808788, + 0x60646B6F, + 0x5A5B5C5D, + 0x585A5B5B, + 0x55555657, + 0x51525253, + 0x50505051, + 0x484C5051, + 0x5B544B47, + 0x78756A5F, + 0x8D827876, + 0x817C8188, + 0x787A8083, + 0x7B787A80, + 0x8D878280, + 0x8E93938F, + 0x82878B8C, + 0x77777D81, + 0x8A8E8C81, + 0x978A8688, + 0x96999F9F, + 0x9593939D, + 0x7A7C7E88, + 0x6D646269, + 0x7D6F686D, + 0x858A8781, + 0x6F757B80, + 0x76736D6B, + 0x61606671, + 0x98887571, + 0x92949395, + 0x75797E81, + 0x70727474, + 0x77777675, + 0x817E7A78, + 0x7A7E8182, + 0x79777576, + 0x807E7D7C, + 0x80818181, + 0x85858280, + 0x81818284, + 0x85848382, + 0x82838485, + 0x83828181, + 0x87868584, + 0x777A7F81, + 0x76757474, + 0x74737374, + 0x76777775, + 0x74747576, + 0x77787876, + 0x71727474, + 0x74747271, + 0x74747474, + 0x76767574, + 0x7F7F7B79, + 0x78797A7D, + 0x76757474, + 0x77777777, + 0x7A7B7B7A, + 0x7B7A7A7A, + 0x79706E71, + 0x78808683, + 0x6F727579, + 0x7975726F, + 0x7C7A7877, + 0x80807F7E, + 0x76787A7C, + 0x7C7A7876, + 0x83817F7F, + 0x7D818485, + 0x7A777574, + 0x7F7E7D7B, + 0x81828383, + 0x81818181, + 0x777B8082, + 0x837F7976, + 0x8F8E8D8C, + 0x8B8C8D8F, + 0x82868A8D, + 0x73767A7F, + 0x65696E71, + 0x5F5F6062, + 0x6863615F, + 0x73716E6B, + 0x69717474, + 0x5C5A5A60, + 0x7E7A716A, + 0x7B7A7B7D, + 0x7277767C, + 0x63625C62, + 0x6E6A6662, + 0x7372706F, + 0x887F7979, + 0x8686898D, + 0x6467737E, + 0x7979746B, + 0x67666C74, + 0x666C706E, + 0x75796857, + 0x5B545462, + 0x5F5C5C5D, + 0x74716B64, + 0x7C7A7877, + 0x777A7D7E, + 0x807A7A7E, + 0x868A8D87, + 0x83818386, + 0x8B8B8986, + 0x76777C81, + 0x7C7D7C79, + 0x7A7F817F, + 0x777B7C7A, + 0x7A747172, + 0x7B808481, + 0x71747577, + 0x7C787471, + 0x7A7B7D7F, + 0x7F7E7B7A, + 0x696B7178, + 0x69666568, + 0x898B7C6C, + 0x93867A7E, + 0x84888B8C, + 0x76787B80, + 0x6E737679, + 0x61626568, + 0x73796F5E, + 0x787B7770, + 0x7E7F7D79, + 0x85878680, + 0x8C888789, + 0x898D908F, + 0x8B8F8D8C, + 0x827A7780, + 0x8F8E8E8E, + 0x9090908F, + 0x84848687, + 0x84858585, + 0x81797E87, + 0x7585928F, + 0x7F786C63, + 0x7F7F7F81, + 0x71747675, + 0x71737371, + 0x7474716E, + 0x7B797775, + 0x6868717A, + 0x66666868, + 0x69666362, + 0x7E7B756F, + 0x8C8B8580, + 0x86858689, + 0x8B898887, + 0x8C8C8C8C, + 0x7A7A818A, + 0x7F7E7E7D, + 0x91877D7A, + 0x6E758391, + 0x78797877, + 0x6E707476, + 0x6E747673, + 0x696C6C6B, + 0x8178706C, + 0x8D8D8D88, + 0x7D82878B, + 0x7F7C7A7A, + 0x918D8887, + 0x747D8990, + 0x817F7A78, + 0x686F7980, + 0x65626263, + 0x6C6E6E6A, + 0x5B5B6065, + 0x575C5E5D, + 0x635C5653, + 0x7474716B, + 0x76757473, + 0x88837C78, + 0x6C6E747E, + 0x6A696B6D, + 0x817B746E, + 0x6B717A80, + 0x6F6F6F70, + 0x74737170, + 0x74747575, + 0x6F707173, + 0x79726F70, + 0x71798080, + 0x76746F6D, + 0x7A7A7A79, + 0x72747778, + 0x6D6C6D6E, + 0x6E6D6C6B, + 0x6E6E6E6E, + 0x6E6E6E6E, + 0x6F6F6E6E, + 0x73747475, + 0x6E6F7072, + 0x6D6A6A6A, + 0x73747370, + 0x7A757271, + 0x8383827F, + 0x77787A7D, + 0x686D7275, + 0x5F5E6164, + 0x5B5E6262, + 0x66635D57, + 0x5C5C5E62, + 0x5B595858, + 0x5155595C, + 0x56504E50, + 0x67625D5B, + 0x747A746D, + 0x68626168, + 0x5B575B62, + 0x746E6862, + 0x75797C7E, + 0x7F7A7674, + 0x75797A7C, + 0x74737272, + 0x827E7874, + 0x82828383, + 0x7E78787A, + 0x7A818785, + 0x73727075, + 0x767A746F, + 0x7A787675, + 0x70747A7C, + 0x73706E6E, + 0x70727475, + 0x7074706B, + 0x6C686669, + 0x696A6C6E, + 0x6467696A, + 0x74706C69, + 0x62666E73, + 0x5F606162, + 0x6261605F, + 0x695D6671, + 0xC7216E61, + 0xEAEBE6DF, + 0x35567ACD, + 0x50565147, + 0x4D51514D, + 0x7367574A, + 0xE912505B, + 0xE8EBE8EB, + 0x755850EB, + 0xE0D12B6C, + 0xE4EBEBD6, + 0x5637252C, + 0x52535860, + 0x68676462, + 0x4C5A666A, + 0xD6A81F1C, + 0xEAE2C5EB, + 0x595B5E61, + 0x605D5B58, + 0xD7356073, + 0xE8E5EBEB, + 0x2BCAEBE1, + 0xCEEBCF1F, + 0x5CE5E8EB, + 0x45584854, + 0x5759534F, + 0x4A624D44, + 0xE52D6867, + 0xE0DCDCEB, + 0x5559505C, + 0x775D4F56, + 0x6351767D, + 0x235E5679, + 0x23201310, + 0x6E553726, + 0x484D5256, + 0x55504A47, + 0x57595A5A, + 0x51525456, + 0x54545352, + 0x4A4C5052, + 0x556B4A47, + 0xE6DC3356, + 0xEBEBE5EA, + 0x1922EBCA, + 0xE8EBE2EB, + 0x4A57E5EB, + 0x504F4E4D, + 0x52525150, + 0xD03C5F54, + 0xE1EBCAD2, + 0x443116E0, + 0x3946403B, + 0x4D4C4E4E, + 0x5D625F56, + 0xDF251610, + 0xE1D5EBE5, + 0x4B505CB5, + 0x4F4D4B41, + 0x53555554, + 0x504F4E50, + 0x504F4E4D, + 0x4E4F5050, + 0xEB254868, + 0xEAD1DFEB, + 0x5A444DEB, + 0x472F4B4E, + 0x54585958, + 0x514F4D4F, + 0xEB1E5555, + 0xE0E3D0DE, + 0x444F5CEB, + 0x2E513B5D, + 0xC9D0E627, + 0x5442E3E7, + 0x5151543D, + 0x4A5E4442, + 0x2B50453D, + 0x1E16102A, + 0x351E1018, + 0x48565444, + 0x44454849, + 0x4E4A4543, + 0x51505357, + 0x50545656, + 0xE0E8EA18, + 0x2EC8E8EB, + 0x69614A36, + 0x53525662, + 0x4D565B54, + 0x2B3D4E50, + 0xEBE4BC42, + 0x5EC3C1D7, + 0x605D4338, + 0xEB153D5D, + 0xE8E8EBE2, + 0x4B4C2F38, + 0x5650444D, + 0x525B5750, + 0xE0E83052, + 0x55E4EBE5, + 0x5A565250, + 0x5E5E5E5C, + 0x4B6B415C, + 0x5852434A, + 0x52395056, + 0xEB326068, + 0xEBEADCE8, + 0x683A6CEB, + 0x585A5C5C, + 0x56565657, + 0x6A5C524C, + 0xCF216E62, + 0xEBEBE9EA, + 0x816E7EE8, + 0x7C69637A, + 0xEBD7C72D, + 0xEBE3E0E9, + 0x7C8768E9, + 0x6E747A80, + 0x76736E6C, + 0x314F7280, + 0x937B5B3A, + 0x7C616168, + 0x54736673, + 0x8C766E6B, + 0x545D7887, + 0x665D625F, + 0x10484A55, + 0xE9C6EBEB, + 0xD6EBE7D5, + 0x69696969, + 0x69696969, + 0x6B6E686A, + 0x725A4D5D, + 0x67656568, + 0x2C695661, + 0x19181110, + 0x7F644125, + 0x63626367, + 0x50564C59, + 0x5348624B, + 0xEBDD1348, + 0x6DDED8E2, + 0x4E444F64, + 0x4E505152, + 0x4A4A4B4C, + 0x434E514F, + 0x38312E35, + 0x7D7A7571, + 0x8482807F, + 0x7C7E7F80, + 0x7B7B7B7C, + 0x807F7E7D, + 0x83828181, + 0x81818182, + 0x81818181, + 0x7F838585, + 0x7474767A, + 0x89837B76, + 0x8C8C8C8C, + 0x8386888A, + 0x81818181, + 0x81828282, + 0x75787B7F, + 0x66676E78, + 0x76716C68, + 0x74747475, + 0x7F7C7A77, + 0x7E7C7C7D, + 0x88878682, + 0x8D8D8D8D, + 0x8083888B, + 0x817D7A7A, + 0x87878683, + 0x8F8D8A87, + 0x878A8D8F, + 0x88898A8B, + 0x8A898988, + 0x79797E83, + 0x8E8C877F, + 0x93918E8C, + 0x98979594, + 0x7D82898E, + 0x7D7D7C7B, + 0x69717778, + 0x65676765, + 0x62626264, + 0x69696865, + 0x6F6D6B6A, + 0x686B6F70, + 0x7D7A736B, + 0x85807A7A, + 0x73797C80, + 0x746D686C, + 0x6D6C6A69, + 0x706E6E6D, + 0x7A777474, + 0x71767C7D, + 0x6D6C6D6E, + 0x6F70706E, + 0x706C6A69, + 0x79797874, + 0x767B7A76, + 0x7B746E6F, + 0x76747474, + 0x7A7A7A7A, + 0x76767676, + 0x76767676, + 0x767D8182, + 0x6667696E, + 0x91938B7B, + 0x7A707687, + 0x84878582, + 0x827F7D7F, + 0x7E7C7B7C, + 0x7E808180, + 0x6D73797C, + 0x736E6968, + 0xA7A6A39F, + 0x9BA2A8A9, + 0x7C767987, + 0x938E827C, + 0x81878D93, + 0x87868381, + 0x9093928D, + 0x7079838B, + 0x746E6D6E, + 0x928F897F, + 0x888D9398, + 0x9D978F89, + 0x878C939A, + 0x88838184, + 0x80818180, + 0x8B86817F, + 0x8A8C8E91, + 0x8B8B8C8B, + 0x81818181, + 0x7D7E7F81, + 0x7E7A7A7B, + 0x6F767E81, + 0x5B5E6367, + 0x58595959, + 0x56575758, + 0x55555656, + 0x50515152, + 0x4D4E4F50, + 0x42474B4C, + 0x4E48413E, + 0x6D6A5F54, + 0x9787756D, + 0x7E7A818C, + 0x84878986, + 0x7A75777C, + 0x9C948A82, + 0x969E9F9E, + 0x81858A8F, + 0x79787C80, + 0x91938F83, + 0x9A908C8D, + 0x999DA1A2, + 0x9B999399, + 0x8483828D, + 0x6E696A71, + 0x7D6F686D, + 0x80878781, + 0x696C6F76, + 0x74706A66, + 0x5F60666F, + 0x8C7C6965, + 0x8588878A, + 0x76797D80, + 0x70727474, + 0x75757574, + 0x86817B77, + 0x81848585, + 0x7E7C7C7E, + 0x8281807F, + 0x82828382, + 0x83838180, + 0x7F7F7F81, + 0x81818181, + 0x81818181, + 0x83818181, + 0x87878584, + 0x7A7C7E80, + 0x7D7C7A7A, + 0x76777877, + 0x75747474, + 0x77757575, + 0x74767879, + 0x6F6F6F6F, + 0x6F6F6F6F, + 0x72717070, + 0x74747473, + 0x80807C7A, + 0x78797A7D, + 0x75757575, + 0x78777675, + 0x837F7B7A, + 0x87888987, + 0x6962676E, + 0x7B818178, + 0x6C6E7274, + 0x74726E6C, + 0x7A777574, + 0x7E7D7D7B, + 0x7476797A, + 0x7A797674, + 0x83807E7E, + 0x85878787, + 0x81818282, + 0x7C7D7F80, + 0x81828282, + 0x81818181, + 0x7C7B7D7F, + 0x89878480, + 0x8C8D8E8F, + 0x8D8C8C8B, + 0x8185898D, + 0x74767A7E, + 0x61676D70, + 0x5E5D5C5D, + 0x625E5C5B, + 0x6E6C6865, + 0x7577726B, + 0x6463666E, + 0x7D7A746E, + 0x79797A7C, + 0x7576747A, + 0x6A696468, + 0x78746E6A, + 0x7B7A7A7A, + 0x8C837E7C, + 0x84868B8E, + 0x5F616E7A, + 0x78787468, + 0x63646E76, + 0x64696C68, + 0x7C7A6855, + 0x5A585E6E, + 0x645D5C5C, + 0x7675736C, + 0x7C7A7877, + 0x7A7D7F7F, + 0x7B747477, + 0x83878A85, + 0x807E8287, + 0x8F8F8D87, + 0x7F7E8388, + 0x81858784, + 0x68727A7E, + 0x73716B66, + 0x7C747170, + 0x767C8382, + 0x676A6F73, + 0x6B6A6867, + 0x76727273, + 0x8181817C, + 0x7176797A, + 0x625E5F68, + 0x7D77706C, + 0x8D8A8681, + 0x84888B8D, + 0x77797C80, + 0x6E707273, + 0x6466686B, + 0x747A705F, + 0x7778746E, + 0x79787774, + 0x8686827C, + 0x8C88898C, + 0x92949592, + 0x8E949697, + 0x817B7A83, + 0x8A888787, + 0x8F8E8D8C, + 0x80818487, + 0x7F818181, + 0x72727F8D, + 0x7A81837C, + 0x8480766F, + 0x787A7D82, + 0x69696B6B, + 0x7276766F, + 0x7A787574, + 0x807E7B7A, + 0x6666707C, + 0x68696B6A, + 0x63656869, + 0x6C696764, + 0x89898681, + 0x81818186, + 0x8481807F, + 0x8E8D8A87, + 0x7F7B8188, + 0x80808182, + 0x93887E7A, + 0x67748794, + 0x72716F6E, + 0x73747473, + 0x6F757674, + 0x6164676A, + 0x7C716761, + 0x8F8E8C86, + 0x7B818689, + 0x7D7B7A79, + 0x88878481, + 0x8B8A8988, + 0x8386888B, + 0x767A7E81, + 0x6E686C71, + 0x767C7F79, + 0x666B6E6F, + 0x5E5D5F62, + 0x6F68615C, + 0x7D7B7874, + 0x80827D76, + 0x8F857B7A, + 0x6A6A6F77, + 0x6A69696B, + 0x7975716E, + 0x7174777A, + 0x7477797A, + 0x6A6B6E71, + 0x6F6D6C6C, + 0x7A787572, + 0x8177767A, + 0x7A858E8D, + 0x72717070, + 0x78777574, + 0x6F727577, + 0x6A6A6B6D, + 0x68696A6B, + 0x62636466, + 0x63636262, + 0x64646463, + 0x73747373, + 0x6E6E7072, + 0x6C6B6C6D, + 0x6E6E6F6E, + 0x78716A65, + 0x8B88847F, + 0x6C727C85, + 0x66686A6A, + 0x5D5E6164, + 0x55585C5E, + 0x67625C59, + 0x65666868, + 0x57555658, + 0x54585C5C, + 0x56504E50, + 0x645F5C59, + 0x6C747470, + 0x6E665E61, + 0x5C565C64, + 0x706E6A65, + 0x6D6F7476, + 0x8881766E, + 0x8D8D8F90, + 0x8184888B, + 0x8B867E77, + 0x7D81868A, + 0x7F7B7A7A, + 0x7F818281, + 0x7A7F7D81, + 0x76756D6D, + 0x82817C79, + 0x74767B80, +}; + +static gctUINT32 ppuMem6[] = +{ + 0x7573706F, + 0x6C707476, + 0x6E6E6A64, + 0x706B6869, + 0x67666768, + 0x61646868, + 0x6868696A, + 0x65666868, + 0x5D5E5F5F, + 0x5F5F5E5D, + 0x5C65635C, + 0xBF245040, + 0xD5DEE0E1, + 0x415C7BBA, + 0x54575652, + 0x4E565A56, + 0x5A47464E, + 0xE3246557, + 0xE4E6E9EA, + 0x715A58E7, + 0xEBE71F6A, + 0xEBE9DFDE, + 0x625D5954, + 0x5C535662, + 0x5D626869, + 0x6264625E, + 0xEBDE254E, + 0xD6E8D9E3, + 0x56595C5F, + 0x5D5C5856, + 0xAB355D5A, + 0xDED5D3C6, + 0x46BBEBDA, + 0xD4EBE019, + 0x5AE5E8EA, + 0x44574752, + 0x5155514F, + 0x3150433D, + 0xB7191D10, + 0xE8E1EBEA, + 0x74655437, + 0x5F575C5D, + 0x694F6365, + 0x43627A7C, + 0x504D5054, + 0x61626059, + 0x4C505456, + 0x56514B49, + 0x54555656, + 0x51515253, + 0x50515151, + 0x4A4C4D4F, + 0x435C5665, + 0xC0B72E45, + 0xEBE4E7EB, + 0x1F46EBD9, + 0xE8EBE2EB, + 0x4A57E5EB, + 0x51515050, + 0x53525252, + 0xD0104F5F, + 0xEBEBEBEB, + 0x25565FBB, + 0x524A6E59, + 0x4A4C4B49, + 0x625E564D, + 0xDD102568, + 0xEBE5E1D2, + 0x45445EEB, + 0x464C454B, + 0x52535352, + 0x54525050, + 0x504F4D4D, + 0x50505050, + 0xC526445B, + 0xEBD6D6D6, + 0x5B594FB7, + 0x3D364A44, + 0x51514A44, + 0x4E4B4A4D, + 0xE7206347, + 0xEAE8D9E5, + 0x3E3D41E8, + 0x25421036, + 0xEBE9EA1D, + 0x6C4CCAE0, + 0x5F4E5344, + 0x505B4D5C, + 0x6E705548, + 0x38434762, + 0x45414848, + 0x4A424A51, + 0x44464747, + 0x4A484444, + 0x52505458, + 0x50545756, + 0xD9EBE910, + 0x47E6E8DC, + 0x3A273956, + 0x4D687863, + 0x4A46474B, + 0x34495652, + 0xEBE7E317, + 0x43E5EBEB, + 0x68605944, + 0xD0125D82, + 0xEBE0E2E9, + 0x3C5C5B4C, + 0x3833444C, + 0x3842504D, + 0xEBE81024, + 0x61C9DAE5, + 0x5758595B, + 0x5E5C5A58, + 0x122A104B, + 0x45251010, + 0x4D455D6A, + 0xB5305453, + 0xE4EBE1EB, + 0x484565C2, + 0x56514E4D, + 0x565A5C5B, + 0x62504E63, + 0xDF45825C, + 0xDEDCE8EB, + 0x807179DD, + 0x5E546893, + 0xEBE6D121, + 0xEBE8E9EB, + 0x988F6DDD, + 0x6F767F84, + 0x77746E6D, + 0x55576081, + 0x626D5646, + 0x805C6172, + 0x2C5D667B, + 0x29102012, + 0x64676E2E, + 0x62595E5A, + 0x10444550, + 0xEBD2E0E2, + 0xD6E2D0EB, + 0x57575757, + 0x57575757, + 0x5A746F61, + 0x7E70554F, + 0x5C56585D, + 0x4977615D, + 0x68594F5B, + 0x6374726A, + 0x615E665C, + 0x4D594B51, + 0x5554672B, + 0xEBDA317A, + 0x4EDFEBEB, + 0x4459553E, + 0x44484B4D, + 0x3A3B3D40, + 0x3F45433E, + 0x38323036, + 0x7777726D, + 0x827B7574, + 0x7A7D8081, + 0x75767779, + 0x817F7D7B, + 0x87878583, + 0x85868787, + 0x82828384, + 0x7D818282, + 0x73737478, + 0x88827B75, + 0x88898A8A, + 0x81848788, + 0x7A7B7C7E, + 0x7C7C7C7B, + 0x7274777A, + 0x6462676E, + 0x79746F6A, + 0x79797A7A, + 0x817F7C7A, + 0x7F7D7D7E, + 0x88888783, + 0x93918F8E, + 0x8C8F9394, + 0x817E7B7A, + 0x87878684, + 0x8D8C8887, + 0x888A8D8E, + 0x8B8C8C8C, + 0x8787888A, + 0x7F7D8184, + 0x92908C85, + 0x8D8D8D8E, + 0x8D8D8E8D, + 0x7A7D848A, + 0x80807D7A, + 0x6E777B7B, + 0x6468696A, + 0x62626466, + 0x61626362, + 0x77726C68, + 0x6E72787A, + 0x817F766E, + 0x817D7A7D, + 0x74757980, + 0x6F68666E, + 0x66676563, + 0x6C686564, + 0x847E7A77, + 0x767D8487, + 0x686B6E71, + 0x6E6C6A68, + 0x706C6969, + 0x797A7975, + 0x797B7A77, + 0x7A767374, + 0x716F6E6F, + 0x75767674, + 0x71706F6F, + 0x78767473, + 0x747B7F81, + 0x6566686E, + 0x82878171, + 0x766A6D7A, + 0x8283817F, + 0x7374797E, + 0x76727070, + 0x7E7E7D7A, + 0x6B70777C, + 0x73706D6A, + 0xA2A19D99, + 0x969DA3A4, + 0x7F7A7A86, + 0x98948780, + 0x86878D92, + 0x87898A88, + 0x878A8986, + 0x7F848787, + 0x938D8987, + 0x999B9B98, + 0x82889198, + 0xA2998D84, + 0x898D939A, + 0x83808085, + 0x827F7E7D, + 0x918E8A86, + 0x81838486, + 0x7E7F8081, + 0x82817E7C, + 0x7A7C8081, + 0x74747477, + 0x656B7274, + 0x57595D60, + 0x57575757, + 0x54555656, + 0x52525354, + 0x4E505050, + 0x4A4A4B4D, + 0x41454848, + 0x4E48413F, + 0x64625B54, + 0x8D806F67, + 0x7C798087, + 0x93928F87, + 0x83818081, + 0xA2978C87, + 0x93999FA3, + 0x7C82898E, + 0x7876797A, + 0x92938F82, + 0x9C938D8C, + 0x969BA2A3, + 0x97938B8E, + 0x807D7B87, + 0x6F6D6F75, + 0x7E726B6E, + 0x7C858682, + 0x6564676F, + 0x736F6761, + 0x6062686F, + 0x8A7D6860, + 0x797B7B83, + 0x76797B7E, + 0x71727374, + 0x73737374, + 0x87817A74, + 0x87878583, + 0x86858586, + 0x86858584, + 0x87878787, + 0x85868584, + 0x80808182, + 0x80818181, + 0x81818180, + 0x83828282, + 0x85858484, + 0x7E7D7E7E, + 0x8382817F, + 0x767C7E7F, + 0x74716F71, + 0x7A777574, + 0x7074797A, + 0x6E6E6D6B, + 0x6B6D6E6E, + 0x74737271, + 0x77777675, + 0x7F7F7C7A, + 0x7576797C, + 0x73747475, + 0x76757474, + 0x857E7976, + 0x747A8387, + 0x615D626A, + 0x7A7B776D, + 0x68696C6E, + 0x726F6B68, + 0x77747372, + 0x7C7B7A79, + 0x73747678, + 0x78767473, + 0x807E7D7C, + 0x84848381, + 0x84868787, + 0x7C7D8081, + 0x81818181, + 0x7E7F8081, + 0x807B7A7B, + 0x8D8D8C87, + 0x8A8C8D8E, + 0x8B8B8989, + 0x7F82878A, + 0x7274797B, + 0x5D646B6F, + 0x5D5C5A5A, + 0x65625F5E, + 0x74716E68, + 0x7C7C756D, + 0x6E6C6E74, + 0x7B7A7774, + 0x74757679, + 0x7B77737A, + 0x73716E72, + 0x81807B78, + 0x7E7F8081, + 0x8B847E7B, + 0x83878C8E, + 0x60616E7B, + 0x797A756A, + 0x62636E79, + 0x686B6C67, + 0x76766C60, + 0x6865656D, + 0x70686565, + 0x7C7F807A, + 0x7D7A7A7A, + 0x7B7D7F7F, + 0x7C767578, + 0x81878985, + 0x76757D85, + 0x8E8D8980, + 0x85878D92, + 0x85878786, + 0x5F6A767D, + 0x6E685F5B, + 0x7C746E6D, + 0x737A8283, + 0x5F636B72, + 0x65656360, + 0x746B6767, + 0x8386857F, + 0x81868481, + 0x635F6371, + 0x70686769, + 0x8083847C, + 0x7C7F8182, + 0x7575777A, + 0x72727170, + 0x6B6D6F71, + 0x767B766A, + 0x74787673, + 0x706F6E6E, + 0x807E7A74, + 0x85838383, + 0x8D8D8B87, + 0x81878D93, + 0x7A747279, + 0x82828181, + 0x7C7D8081, + 0x7C7C8185, + 0x7D7F817F, + 0x686F8292, + 0x7F7E786E, + 0x87888581, + 0x72757A82, + 0x74787773, + 0x70767975, + 0x7E7B7976, + 0x87858180, + 0x6B687380, + 0x6B6E7271, + 0x68676768, + 0x7E7A746D, + 0x87888784, + 0x7D7D7F83, + 0x7A7A7B7B, + 0x83817F7C, + 0x807B7D83, + 0x7C7E8183, + 0x8B817B7A, + 0x6371838D, + 0x6E6D6C6C, + 0x7271706E, + 0x6E737372, + 0x62636669, + 0x756C625D, + 0x8D89847D, + 0x7B808487, + 0x7C7A7979, + 0x75797A7A, + 0x93887B74, + 0x81858D92, + 0x8482817F, + 0x7A75787C, + 0x81878984, + 0x6F797D7D, + 0x63605E63, + 0x817C746E, + 0x8A878583, + 0x8785807B, + 0x86858687, + 0x7271757B, + 0x706F7072, + 0x75726F6D, + 0x7A7A7978, + 0x767A7C7D, + 0x62656B71, + 0x69666463, + 0x807B756F, + 0x847A7D84, + 0x7E8A9491, + 0x70717475, + 0x77767472, + 0x6D707477, + 0x68696A6B, + 0x63686C6E, + 0x56575A5E, + 0x59585858, + 0x5B5B5A5A, + 0x72706E6D, + 0x6E6F7172, + 0x6C6C6E6F, + 0x696B6D6D, + 0x716B645F, + 0x8A847C76, + 0x666F7F8B, + 0x5F626263, + 0x605E5F61, + 0x5C5E6161, + 0x645E5B59, + 0x67696B69, + 0x5251575D, + 0x55595B58, + 0x55504F50, + 0x625D5A58, + 0x676F7270, + 0x71675D5D, + 0x5B565C66, + 0x69696863, + 0x686A6E70, + 0x8B82756C, + 0xA09E9B9A, + 0x8A929BA0, + 0x81817D7A, + 0x7C7C7D80, + 0x7E7F7C7A, + 0x82807C7C, + 0x828C8A8B, + 0x716E676D, + 0x85847D76, + 0x74757A80, + 0x76736F6E, + 0x6A6E7477, + 0x6E706C67, + 0x6B676468, + 0x65646362, + 0x68676665, + 0x67696A69, + 0x64626263, + 0x62626262, + 0x63636262, + 0x644F5468, + 0x2A10827B, + 0x2D211F20, + 0x5350443E, + 0x5A58544F, + 0x50585E5C, + 0x6B615447, + 0xEA104450, + 0xEBE9EBE1, + 0x575F79EB, + 0xE5D9106C, + 0xE2EBE5E4, + 0x5063746E, + 0x66534D4F, + 0x5B626666, + 0x56585756, + 0xEBEB1A69, + 0xE3EBEBD8, + 0x54585D61, + 0x59565453, + 0x30165A5A, + 0x10101B21, + 0x492D1F1C, + 0xE2E0DB26, + 0x59E6E9EA, + 0x43584850, + 0x4E51504E, + 0x1C443F3B, + 0xEBD9E9E8, + 0x1531C7E7, + 0x6758683B, + 0x4C5E694B, + 0x62595C64, + 0x626C4C45, + 0x56494A50, + 0x46576865, + 0x4F525659, + 0x5755504E, + 0x52535354, + 0x52525252, + 0x50505151, + 0x4E4E4E4F, + 0x5C5C3F39, + 0x10301252, + 0x12101E10, + 0x15392A30, + 0xE8EBE2EB, + 0x4A57E5EB, + 0x53535353, + 0x53535353, + 0xE010544E, + 0xEAE3EBEA, + 0x5A533CEB, + 0x3E464447, + 0x4E4F4B47, + 0x4D4C4B4C, + 0xD6184E4F, + 0xE8E5E6DC, + 0x47304AE4, + 0x4D524C4A, + 0x50515252, + 0x51505050, + 0x4F4E4D4D, + 0x50505050, + 0x35114B4F, + 0x1019251F, + 0x637F4C48, + 0x434E513C, + 0x544F463F, + 0x4D4E5053, + 0xDD256E3D, + 0xDEEBEBEB, + 0x584F47EA, + 0xE6D02543, + 0xE2EBEBB9, + 0x323C3126, + 0x4C284458, + 0x30333C5F, + 0x4949243C, + 0x3A573E4A, + 0x464F6056, + 0x58373D4F, + 0x49484645, + 0x5955504B, + 0x51515559, + 0x4C505554, + 0xDEE0E842, + 0x51E8E6E7, + 0x5A514842, + 0x4B515A5D, + 0x3C34494F, + 0x623E4453, + 0xD3D9EB1C, + 0x3EDFD1DC, + 0x50555F38, + 0xEB2F6071, + 0xE9E3DAE6, + 0x31433F2D, + 0x59585647, + 0x3A3D4253, + 0xEAE71666, + 0x87E7E5E8, + 0x5A5F666A, + 0x615E5A58, + 0xEBD1335E, + 0xB2DCE6EB, + 0x515E7380, + 0x491C4B4A, + 0x2A2C1310, + 0x3654442C, + 0x50494648, + 0x4F565B59, + 0x7E989D95, + 0xE8103F47, + 0xEBE9EBE2, + 0x78717AE8, + 0x615A6E94, + 0xE4E3D624, + 0x2A29ABEB, + 0x84784C2C, + 0x757B8186, + 0x7A787474, + 0x5A535B70, + 0x896F5C5B, + 0x775E6978, + 0x15576876, + 0xE1E2EBEB, + 0x4F6BA3BB, + 0x695C6260, + 0x1250565E, + 0xEBE8E8EB, + 0x102A4DDE, + 0x5B5B5B5B, + 0x5B5B5B5B, + 0x66595D4F, + 0x4E56695B, + 0x4E48494B, + 0x3D5C4D4A, + 0x666C583E, + 0x61767564, + 0x2F4C5B68, + 0x5A553528, + 0x30445953, + 0xDFEB183F, + 0x59D3E3DE, + 0x3E3D4147, + 0x3E424547, + 0x3334373A, + 0x3B3E3833, + 0x3E383436, + 0x6E6D6761, + 0x8078706E, + 0x7A7E8182, + 0x73747477, + 0x817F7C7A, + 0x87878684, + 0x86878787, + 0x81818284, + 0x7D818181, + 0x75747679, + 0x87827C77, + 0x83848687, + 0x7D808284, + 0x7475777A, + 0x77777675, + 0x70727476, + 0x68626268, + 0x7D79746E, + 0x83828180, + 0x86868584, + 0x86838383, + 0x8D8D8C89, + 0x8B89898A, + 0x888A8C8C, + 0x82807E7D, + 0x87878684, + 0x8D8B8886, + 0x8A8B8D8D, + 0x8C8D8D8C, + 0x8787898B, + 0x827F8081, + 0x9493918A, + 0x86878C92, + 0x81868988, + 0x74767D83, + 0x82817E79, + 0x7B81817E, + 0x686E7477, + 0x63636567, + 0x64656665, + 0x7A746E6A, + 0x6E747A7D, + 0x82817A73, + 0x7D7A7A7D, + 0x7473767D, + 0x6E686971, + 0x61626362, + 0x68645F5E, + 0x8D867E7A, + 0x7C858D91, + 0x686E7477, + 0x6E6B6867, + 0x726F6E6E, + 0x77787875, + 0x75757677, + 0x75767776, + 0x6C696869, + 0x7374736F, + 0x6E6C6C6C, + 0x7B797471, + 0x7075797A, + 0x6565676B, + 0x747C7A6C, + 0x7367656D, + 0x81807D7B, + 0x656D777F, + 0x716C6868, + 0x7E7E7B77, + 0x6F72777C, + 0x77767470, + 0x99999590, + 0x8F969B9B, + 0x7F79767F, + 0x96958A81, + 0x8987878B, + 0x81878D8D, + 0x7E7F7F7D, + 0x93928B82, + 0xA8A5A2A1, + 0xA6A8ABAB, + 0x80858D94, + 0xA2998D82, + 0x85898F95, + 0x7C7A7A81, + 0x88828181, + 0x93949490, + 0x7A7B7A7A, + 0x71727478, + 0x807C7976, + 0x7A7B7F81, + 0x6C6F7478, + 0x6063686A, + 0x5556595C, + 0x56565656, + 0x51535454, + 0x4D4E4F50, + 0x4B4D4E4E, + 0x4848494A, + 0x44474847, + 0x504A4442, + 0x5E5B5857, + 0x78736B63, + 0x83807E7D, + 0x86898A87, + 0x8C8E8B88, + 0x968B8185, + 0x898C949D, + 0x787F878A, + 0x7A777878, + 0x90938E83, + 0xA098908A, + 0x949BA3A5, + 0x999A9396, + 0x71717385, + 0x72707175, + 0x7E746E70, + 0x767E807F, + 0x605D5F69, + 0x746F655C, + 0x62666C71, + 0x93897166, + 0x7C7C7C87, + 0x77797A7B, + 0x72727374, + 0x72737475, + 0x85807874, + 0x8A888683, + 0x87878889, + 0x8C8C8C8C, + 0x8D8D8D8C, + 0x898A8A89, + 0x84848486, + 0x82828384, + 0x84838282, + 0x85858686, + 0x82838384, + 0x807F7D7D, + 0x85848381, + 0x767E8385, + 0x74706E6F, + 0x7C797574, + 0x6D72797C, + 0x716E6C69, + 0x696C6E71, + 0x7A787574, + 0x7C7C7C7B, + 0x7D7D7B79, + 0x7374767A, + 0x70717273, + 0x7271706F, + 0x78777676, + 0x5C656F76, + 0x62616367, + 0x74716C66, + 0x64636566, + 0x716E6A66, + 0x74727070, + 0x7B7A7876, + 0x70727476, + 0x76747270, + 0x7B7A7A79, + 0x81807E7C, + 0x81818181, + 0x7D7E8081, + 0x7F807F7F, + 0x7A7B7D7F, + 0x817A7877, + 0x8D8E8E89, + 0x898A8A8A, + 0x87888889, + 0x7B7F8487, + 0x70737578, + 0x5C62696E, + 0x5E5C5958, + 0x6D686664, + 0x7C7A7671, + 0x7C7F7B76, + 0x74717175, + 0x76777675, + 0x6F6F7174, + 0x7E746E76, + 0x7C7B777A, + 0x86878583, + 0x7A7A7D81, + 0x8A857E7A, + 0x868B8E8E, + 0x6365727E, + 0x7778746C, + 0x5F616B75, + 0x696C6C65, + 0x71736E6A, + 0x78736E6E, + 0x7E777474, + 0x82878986, + 0x7E7C7B7B, + 0x797B7F80, + 0x7A737274, + 0x81868884, + 0x6D6E7781, + 0x89878175, + 0x8D908E8C, + 0x82828489, + 0x676F797E, + 0x6A656060, + 0x7A706A68, + 0x727A8181, + 0x6666686B, + 0x68696B69, + 0x74686261, + 0x85878781, + 0x8C8F8C88, + 0x69676D7D, + 0x716C6C6E, + 0x7A7D7E79, + 0x73737474, + 0x76757474, + 0x7B7A7776, + 0x78797A7B, + 0x777A7B78, + 0x6F767A77, + 0x6C6A6A6B, + 0x7674706E, + 0x7E7E7A77, + 0x86827E7D, + 0x7B7E8287, + 0x68686D75, + 0x7A7A7A79, + 0x6E717478, + 0x7A77797A, + 0x8182817E, + 0x68728391, + 0x807A7168, + 0x81858787, + 0x6F71757A, + 0x838B8476, + 0x7276787A, + 0x7D7A7674, + 0x89878380, + 0x736F7884, + 0x6E737979, + 0x72696462, + 0x9F988C7E, + 0x84878787, + 0x7B7C7E81, + 0x7475787A, + 0x76757473, + 0x817B7B80, + 0x797C8185, + 0x7A777778, + 0x67717B7E, + 0x6C6B6B6C, + 0x71706E6D, + 0x6A6E6F71, + 0x63626266, + 0x6E696461, + 0x84807973, + 0x7A7E8285, + 0x7A7A7979, + 0x7A80817E, + 0x90857A76, + 0x7A81898F, + 0x83817D7A, + 0x84818182, + 0x8A8B8A87, + 0x717D8384, + 0x6C656165, + 0x8B867E78, + 0x94928E8D, + 0x8680848B, + 0x7C879290, + 0x7D7D8085, + 0x7877787C, + 0x7473706E, + 0x7A797877, + 0x74777979, + 0x5E62686E, + 0x63605E5E, + 0x7B787169, + 0x7E777B83, + 0x818B918B, + 0x7072777B, + 0x78777472, + 0x6B6E7477, + 0x68696A6A, + 0x60666A6D, + 0x5052565A, + 0x53525151, + 0x56555554, + 0x706D6A68, + 0x70717271, + 0x6D6E6F71, + 0x686A6C6D, + 0x69686664, + 0x7A746E6A, + 0x646C7982, + 0x5C5E6061, + 0x64626060, + 0x63656666, + 0x625D5C5C, + 0x66696B68, + 0x50525C64, + 0x55575754, + 0x55505051, + 0x605C5857, + 0x676D6D6A, + 0x6E675F5F, + 0x5A545B64, + 0x68696863, + 0x69696C6D, + 0x857E746C, + 0x928F8F8F, + 0x848A9093, + 0x71757A7E, + 0x7F7A736F, + 0x8082807B, + 0x847F7A7A, + 0x86918E8E, + 0x6868636E, + 0x7F7B726A, + 0x7776777B, + 0x78746F6D, + 0x6C71777A, + 0x6E74736F, + 0x625F5E65, + 0x62656461, + 0x66615C5D, + 0x6D6E6862, + 0x63606065, + 0x68686867, + 0x69696968, + 0x585F5652, + 0x76466A54, + 0x565A6671, + 0x7372715F, + 0x645F5449, + 0x52585F63, + 0x64444043, + 0xEB115C68, + 0xE2DCEBE2, + 0x665766D8, + 0xEBEB1369, + 0xE5EBE9E8, + 0x50545543, + 0x5350585A, + 0x5E605D58, + 0x5C5D5D5C, + 0xE0E8106E, + 0xEBE2EBDD, + 0x53596064, + 0x53515050, + 0x4A3E604D, + 0x6C66624F, + 0x594D4F4B, + 0xE6EBEA1D, + 0x58E7EBEB, + 0x43594950, + 0x4E4F4F4E, + 0x1344443F, + 0xE6EBEAEB, + 0x6A68E8E3, + 0x2C105058, + 0x12182512, + 0x15101918, + 0x52581C1A, + 0x4D565550, + 0x6B594744, + 0x484A5157, + 0x4D4D4C4A, + 0x50505152, + 0x50504F4F, + 0x50505151, + 0x53525150, + 0x56565660, + 0x64694B5B, + 0x5F656E5D, + 0x374B484A, + 0xE8EBE2EB, + 0x4A57E5EB, + 0x54545454, + 0x53535353, + 0xEB2D6C44, + 0xEBE5EBE2, + 0x304547DA, + 0x48394743, +}; + +static gctUINT32 ppuMem7[] = +{ + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + +#define PATCH_OFFSET_0 29 +#define PATCH_OFFSET_1 31 +#define PATCH_OFFSET_2 83 +/* Buffer[3] is not used. */ +#define PATCH_OFFSET_3 +#define PATCH_OFFSET_4 11 +#define PATCH_OFFSET_5 17 +#define PATCH_OFFSET_6 23 +#define PATCH_OFFSET_7 33 +#endif + +#if gcdRESET_USC +static gctUINT32 nnBuffer0[] = { + 0x8010010c, + 0x07383fc2, + 0xc1000000, + 0x00a01000, + 0x08e00008, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000800, + 0x40000000, + 0x00e700ff, + 0x00000040, + 0x00000000, + 0x03ffffff, + 0x00000000, + 0x03ffffff, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000 +}; + +static gctUINT32 nnBuffer1[] = { + 0x8040010c, + 0x07383fd5, + 0x01000000, + 0x00a00e00, + 0x08700008, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000800, + 0x00000000, + 0x00e701fe, + 0x00000070, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000 +}; + +static gctUINT32 nnBuffer2[] = { + 0x8080010c, + 0x07383fd5, + 0x01000000, + 0x00a01000, + 0x09000008, + 0x00000000, + 0x00000000, + 0x00000000 +}; + +#define NN_BUFFER_IDX 0 +#define KERNEL_BUFFER_IDX 1 +#define INPUT_BUFFER_IDX 2 +#define RESULT_BUFFER_IDX 3 + +#define PATCH_KERNEL_OFFSET 5 +#define PATCH_INPUT_OFFSET 6 +#define PATCH_RESULT_OFFSET 7 + +static gctUINT32 instBuffer[] = { + 0x04021009, + 0x00200000, + 0x80000000, + 0x20000048, + 0x02021009, + 0x00200000, + 0x80000000, + 0x20154048, + 0x07801009, + 0x00200000, + 0x80000000, + 0x20390008, + 0x07801033, + 0x3fe02800, + 0x81540140, + 0x00390008, + 0x02021001, + 0x2aa02800, + 0x80000000, + 0x202a8048, + 0x07801033, + 0x3fe02800, + 0x81540140, + 0x00390008, + 0x02021001, + 0x2aa02800, + 0x80000000, + 0x202a8048, + 0x07801033, + 0x3fe02800, + 0x81540140, + 0x00390008, + 0x02021001, + 0x2aa02800, + 0x80000000, + 0x202a8048, + 0x07801033, + 0x3fe02800, + 0x81540140, + 0x00390008, + 0x02021001, + 0x2aa02800, + 0x80000000, + 0x202a8048, + 0x07801033, + 0x3fe02800, + 0x81540140, + 0x00390008, + 0x02021001, + 0x2aa02800, + 0x80000000, + 0x202a8048, + 0x07801033, + 0x3fe02800, + 0x81540140, + 0x00390008 +}; + +#define INST_BUFFER_IDX 0 +#define OUTPUT_BUFFER_IDX 1 +#endif + +#define _GC_OBJ_ZONE gcvZONE_HARDWARE + +#define gcmSEMAPHORESTALL(buffer) \ + do \ + { \ + /* Arm the PE-FE Semaphore. */ \ + *buffer++ \ + = gcmSETFIELDVALUE(0, AQ_COMMAND_LOAD_STATE_COMMAND, OPCODE, LOAD_STATE) \ + | gcmSETFIELD (0, AQ_COMMAND_LOAD_STATE_COMMAND, COUNT, 1) \ + | gcmSETFIELD (0, AQ_COMMAND_LOAD_STATE_COMMAND, ADDRESS, 0x0E02); \ + \ + *buffer++ \ + = gcmSETFIELDVALUE(0, AQ_SEMAPHORE, SOURCE, FRONT_END) \ + | gcmSETFIELDVALUE(0, AQ_SEMAPHORE, DESTINATION, PIXEL_ENGINE);\ + \ + /* STALL FE until PE is done flushing. */ \ + *buffer++ \ + = gcmSETFIELDVALUE(0, STALL_COMMAND, OPCODE, STALL); \ + \ + *buffer++ \ + = gcmSETFIELDVALUE(0, STALL_STALL, SOURCE, FRONT_END) \ + | gcmSETFIELDVALUE(0, STALL_STALL, DESTINATION, PIXEL_ENGINE); \ + } while(0) + +static gceSTATUS +_FuncExecute(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + return gckHARDWARE_ExecuteFunctions(Execution); +} + +static gceSTATUS +_FuncValidate_MMU(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + if ((hardware->mmuVersion > 0) && + hardware->options.enableMMU && + (hardware->options.secureMode != gcvSECURE_IN_TA)) + { + Execution->valid = gcvTRUE; + } + else + { + Execution->valid = gcvFALSE; + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FuncRelease_MMU(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + if (Execution->funcVidMem) + { + if (Execution->logical) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->funcVidMem, + 0, + gcvFALSE, + gcvFALSE + )); + Execution->logical = gcvNULL; + } + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->funcVidMem + )); + Execution->funcVidMem = gcvNULL; + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_ProgramMMUStates( + IN gckHARDWARE Hardware, + IN gckMMU Mmu, + IN gceMMU_MODE Mode, + IN gctPOINTER Logical, + IN OUT gctUINT32 * Bytes + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 config, address; + gctUINT32 extMtlb, extSafeAddress, configEx = 0; + gctPHYS_ADDR_T physical; + gctUINT32_PTR buffer; + gctBOOL ace; + gctUINT32 reserveBytes = 0; + + gctBOOL config2D; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(Hardware->mmuVersion != 0); + + ace = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_ACE); + + switch (Hardware->options.secureMode) + { + case gcvSECURE_IN_NORMAL: + reserveBytes = 8 + 4 * 4; + break; + case gcvSECURE_NONE: + reserveBytes = 16 + 4 * 4; + if (ace) + { + reserveBytes += 8; + } + break; + case gcvSECURE_IN_TA: + default: + gcmkASSERT(gcvFALSE); + gcmkPRINT("%s(%d): secureMode is wrong", __FUNCTION__, __LINE__); + break; + } + + config2D = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_PIPE_3D) + && gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_PIPE_2D); + + if (config2D) + { + reserveBytes += + /* Pipe Select. */ + 4 * 4 + /* Configure MMU States. */ + + 4 * 4 + /* Semaphore stall */ + + 4 * 8; + + if (ace) + { + reserveBytes += 8; + } + } + + reserveBytes += 8; + + physical = Mmu->mtlbPhysical; + + config = (gctUINT32)(physical & 0xFFFFFFFF); + extMtlb = (gctUINT32)(physical >> 32); + + /* more than 40bit physical address */ + if (extMtlb & 0xFFFFFF00) + { + gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); + } + + physical = Mmu->safePagePhysical; + + address = (gctUINT32)(physical & 0xFFFFFFFF); + extSafeAddress = (gctUINT32)(physical >> 32); + + if (address & 0x3F) + { + gcmkONERROR(gcvSTATUS_NOT_ALIGNED); + } + + /* more than 40bit physical address */ + if (extSafeAddress & 0xFFFFFF00) + { + gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); + } + + if (ace) + { + configEx = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (extSafeAddress) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (extMtlb) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))); + } + + switch (Mode) + { + case gcvMMU_MODE_1K: + if (config & 0x3FF) + { + gcmkONERROR(gcvSTATUS_NOT_ALIGNED); + } + + config |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + break; + + case gcvMMU_MODE_4K: + if (config & 0xFFF) + { + gcmkONERROR(gcvSTATUS_NOT_ALIGNED); + } + + config |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + break; + + default: + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + if (Logical != gcvNULL) + { + buffer = (gctUINT32_PTR)Logical; + + if (Hardware->options.secureMode == gcvSECURE_IN_NORMAL) + { + gcsMMU_TABLE_ARRAY_ENTRY * entry; + entry = (gcsMMU_TABLE_ARRAY_ENTRY *) Hardware->pagetableArray.logical; + + /* Setup page table array entry. */ + if (Hardware->bigEndian) + { + entry->low = gcmBSWAP32(config); + entry->high = gcmBSWAP32(extMtlb); + } + else + { + entry->low = config; + entry->high = extMtlb; + } + + gcmkDUMP(Mmu->os, "#[mmu: page table array]"); + + gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08X]", + (unsigned long long)Hardware->pagetableArray.address, entry->low, 4); + + gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08X]", + (unsigned long long)Hardware->pagetableArray.address + 4, entry->high, 4); + + gcmkVERIFY_OK(gckVIDMEM_NODE_CleanCache( + Hardware->kernel, + Hardware->pagetableArray.videoMem, + 0, + entry, + 8 + )); + + /* Setup command buffer to load index 0 of page table array. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x006B) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ + = (((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) &((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16)))); + } + else + { + gcmkASSERT(Hardware->options.secureMode == gcvSECURE_NONE); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0061) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = config; + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0060) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = address; + + if (ace) + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0068) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ + = configEx; + } + } + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E12) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))); + + do{*buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8)));} while(0); +; + + + if (config2D) + { + /* LoadState(AQPipeSelect, 1), pipe. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E00) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = 0x1; + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0061) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = config; + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0060) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = address; + + if (ace) + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0068) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ + = configEx; + } + + do{*buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8)));} while(0); +; + + + /* LoadState(AQPipeSelect, 1), pipe. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E00) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = 0x0; + + do{*buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8))); *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 12:8) - (0 ? + 12:8) + 1))))))) << (0 ? + 12:8))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 12:8) - (0 ? + 12:8) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 12:8) - (0 ? 12:8) + 1))))))) << (0 ? 12:8)));} while(0); +; + + } + + } + + if (Bytes != gcvNULL) + { + *Bytes = reserveBytes; + } + + /* Return the status. */ + gcmkFOOTER_NO(); + return status; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_ProgramMMUStatesMCFE( + IN gckHARDWARE Hardware, + IN gckMMU Mmu, + IN gceMMU_MODE Mode, + IN gctPOINTER Logical, + IN OUT gctUINT32 * Bytes + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 config, address; + gctUINT32 extMtlb, extSafeAddress, configEx = 0; + gctPHYS_ADDR_T physical; + gctUINT32_PTR buffer; + gctBOOL ace; + gctUINT32 reserveBytes = 0; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(Hardware->mmuVersion != 0); + + ace = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_ACE); + + switch (Hardware->options.secureMode) + { + case gcvSECURE_IN_NORMAL: + reserveBytes = 8; + reserveBytes += 8; + break; + case gcvSECURE_NONE: + reserveBytes = 16;; + if (ace) + { + reserveBytes += 8; + reserveBytes += 8; + } + break; + case gcvSECURE_IN_TA: + default: + gcmkASSERT(gcvFALSE); + gcmkPRINT("%s(%d): secureMode is wrong", __FUNCTION__, __LINE__); + break; + } + + physical = Mmu->mtlbPhysical; + + config = (gctUINT32)(physical & 0xFFFFFFFF); + extMtlb = (gctUINT32)(physical >> 32); + + /* more than 40bit physical address */ + if (extMtlb & 0xFFFFFF00) + { + gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); + } + + physical = Mmu->safePagePhysical; + + address = (gctUINT32)(physical & 0xFFFFFFFF); + extSafeAddress = (gctUINT32)(physical >> 32); + + if (address & 0x3F) + { + gcmkONERROR(gcvSTATUS_NOT_ALIGNED); + } + + /* more than 40bit physical address */ + if (extSafeAddress & 0xFFFFFF00) + { + gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); + } + + if (ace) + { + configEx = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (extSafeAddress) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) (extMtlb) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))); + } + + switch (Mode) + { + case gcvMMU_MODE_1K: + if (config & 0x3FF) + { + gcmkONERROR(gcvSTATUS_NOT_ALIGNED); + } + + config |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + break; + + case gcvMMU_MODE_4K: + if (config & 0xFFF) + { + gcmkONERROR(gcvSTATUS_NOT_ALIGNED); + } + + config |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + break; + + default: + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + if (Logical != gcvNULL) + { + buffer = Logical; + + if (Hardware->options.secureMode == gcvSECURE_IN_NORMAL) + { + gcsMMU_TABLE_ARRAY_ENTRY * entry; + entry = (gcsMMU_TABLE_ARRAY_ENTRY *) Hardware->pagetableArray.logical; + + /* Setup page table array entry. */ + if (Hardware->bigEndian) + { + entry->low = gcmBSWAP32(config); + entry->high = gcmBSWAP32(extMtlb); + } + else + { + entry->low = config; + entry->high = extMtlb; + } + + gcmkDUMP(Mmu->os, "#[mmu: page table array]"); + + gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08X]", + (unsigned long long)Hardware->pagetableArray.address, entry->low, 4); + + gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08X]", + (unsigned long long)Hardware->pagetableArray.address + 4, entry->high, 4); + + gcmkVERIFY_OK(gckVIDMEM_NODE_CleanCache( + Hardware->kernel, + Hardware->pagetableArray.videoMem, + 0, + entry, + 8 + )); + + /* Setup command buffer to load index 0 of page table array. */ + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x006B) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ + = (((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) &((((gctUINT32) (~0U)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16)))); + + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + } + else + { + gcmkASSERT(Hardware->options.secureMode == gcvSECURE_NONE); + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0061) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = config; + + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0060) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = address; + + if (ace) + { + *buffer++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0068) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *buffer++ = configEx; + + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + *buffer++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + } + } + } + + if (Bytes != gcvNULL) + { + *Bytes = reserveBytes; + } + + /* Return the status. */ + gcmkFOOTER_NO(); + return status; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_FuncInit_MMU(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gceSTATUS status; + gctUINT32 mmuBytes = 0; + gctUINT32 tailBytes; + gctUINT32 flags = gcvALLOC_FLAG_CONTIGUOUS; + gceMMU_MODE mode; + gcePOOL pool; + gctPHYS_ADDR_T physical; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + +#if gcdENABLE_MMU_1KMODE + mode = gcvMMU_MODE_1K; +#else + mode = gcvMMU_MODE_4K; +#endif + + flags |= gcvALLOC_FLAG_4GB_ADDR; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + flags |= gcvALLOC_FLAG_CACHEABLE; +#endif + +#if !gcdCAPTURE_ONLY_MODE + pool = gcvPOOL_DEFAULT; +#else + pool = gcvPOOL_VIRTUAL; +#endif + + Execution->funcVidMemBytes = 1024; + /* Allocate mmu command buffer within 32bit space */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + hardware->kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + flags, + &Execution->funcVidMemBytes, + &pool, + &Execution->funcVidMem + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + hardware->kernel, + Execution->funcVidMem, + gcvFALSE, + gcvFALSE, + &Execution->logical + )); + + /* Get CPU physical address. */ + gcmkONERROR(gckVIDMEM_NODE_GetPhysical( + hardware->kernel, + Execution->funcVidMem, + 0, + &physical + )); + + /* Convert to GPU physical address. */ + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( + hardware->os, + physical, + &physical + )); + + if (physical & 0xFFFFFFFF00000000ULL) + { + gcmkFATAL("%s(%d): Command buffer physical address (0x%llx) for MMU setup exceeds 32bits, " + "please rebuild kernel with CONFIG_ZONE_DMA32=y or CONFIG_ZONE_DMA=y or both.", + __FUNCTION__, __LINE__, physical); + + gcmkFATAL("Some Archs, for ARM64, the setting is special:\n" + "kernel version ZONE_DMA ZONE_DMA32\n" + "3.7 - 3.14 no yes\n" + "3.15 - 4.15 yes no\n" + "4.16 - 5.4.5 no yes\n" + "5.5.rc1 - yes yes\n"); + + gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY); + } + + gcmkSAFECASTPHYSADDRT(Execution->address, physical); + + if (hardware->mcFE) + { + gcmkONERROR(_ProgramMMUStatesMCFE( + hardware, + hardware->kernel->mmu, + mode, + Execution->logical, + &mmuBytes + )); + } + else + { + gcmkONERROR(_ProgramMMUStates( + hardware, + hardware->kernel->mmu, + mode, + Execution->logical, + &mmuBytes + )); + } + + Execution->endAddress = Execution->address + mmuBytes; + Execution->endLogical = (gctUINT8_PTR)Execution->logical + mmuBytes; + + if (hardware->wlFE) + { + tailBytes = (gctUINT32)(Execution->funcVidMemBytes - mmuBytes); + gcmkONERROR(gckWLFE_End(hardware, Execution->endLogical, Execution->endAddress, &tailBytes)); + } + else + { + tailBytes = 0; + } + + Execution->bytes = mmuBytes + tailBytes; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + hardware->kernel, + Execution->funcVidMem, + 0, + Execution->logical, + Execution->bytes + )); + + return gcvSTATUS_OK; + +OnError: + _FuncRelease_MMU(Execution); + + return status; +} + +static gceSTATUS +_FuncExecute_MMU(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 address = 0; + gctUINT32 idle; + gctUINT32 timer = 0, delay = 1; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + gckMMU mmu = hardware->kernel->mmu; + + /* Prepared command sequence contains an END, + ** so update lastEnd and store executeCount to END command. + */ + gctUINT32_PTR endLogical = (gctUINT32_PTR)Execution->endLogical; + + hardware->lastEnd = Execution->endAddress; + + if (hardware->wlFE) + { + /* Append a executeCount in End command, MCFE does not support such End command. */ + *(endLogical + 1) = hardware->executeCount + 1; + } + + if (hardware->options.secureMode == gcvSECURE_IN_NORMAL) + { + gctUINT32 extSafeAddress; + /* Set up base address of page table array. */ + gcmkONERROR(gckOS_WriteRegisterEx( + hardware->os, + hardware->core, + 0x0038C, + (gctUINT32)(hardware->pagetableArray.address & 0xFFFFFFFF) + )); + + gcmkONERROR(gckOS_WriteRegisterEx( + hardware->os, + hardware->core, + 0x00390, + (gctUINT32)((hardware->pagetableArray.address >> 32) & 0xFFFFFFFF) + )); + + gcmkONERROR(gckOS_WriteRegisterEx( + hardware->os, + hardware->core, + 0x00394, + 1 + )); + + address = (gctUINT32)(mmu->safePagePhysical & 0xFFFFFFFF); + extSafeAddress = (gctUINT32)(mmu->safePagePhysical >> 32); + + if (address & 0x3F) + { + gcmkONERROR(gcvSTATUS_NOT_ALIGNED); + } + + /* more than 40bit physical address */ + if (extSafeAddress & 0xFFFFFF00) + { + gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); + } + + gcmkONERROR(gckOS_WriteRegisterEx( + hardware->os, + hardware->core, + 0x0039C, + address + )); + + gcmkONERROR(gckOS_WriteRegisterEx( + hardware->os, + hardware->core, + 0x00398, + address + )); + + gcmkONERROR(gckOS_WriteRegisterEx( + hardware->os, + hardware->core, + 0x003A0, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 23:16) - (0 ? + 23:16) + 1))))))) << (0 ? + 23:16))) | (((gctUINT32) ((gctUINT32) ((gctUINT32)extSafeAddress) & ((gctUINT32) ((((1 ? + 23:16) - (0 ? + 23:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 23:16) - (0 ? 23:16) + 1))))))) << (0 ? 23:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:31) - (0 ? + 31:31) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:31) - (0 ? + 31:31) + 1))))))) << (0 ? + 31:31))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 31:31) - (0 ? + 31:31) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:31) - (0 ? 31:31) + 1))))))) << (0 ? 31:31))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) ((gctUINT32)extSafeAddress) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:15) - (0 ? + 15:15) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:15) - (0 ? + 15:15) + 1))))))) << (0 ? + 15:15))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 15:15) - (0 ? + 15:15) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:15) - (0 ? 15:15) + 1))))))) << (0 ? 15:15))) + )); + } + + gckFUNCTION_Dump(Execution); + + /* Execute prepared command sequence. */ + if (hardware->mcFE) + { + gcmkONERROR(gckMCFE_Execute( + hardware, + gcvFALSE, + 0, + Execution->address, + Execution->bytes + )); + } + else + { + gcmkONERROR(gckWLFE_Execute( + hardware, + Execution->address, + Execution->bytes + )); + } + +#if gcdLINK_QUEUE_SIZE + { + gcuQUEUEDATA data; + + gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); + + data.linkData.start = Execution->address; + data.linkData.end = Execution->address + Execution->bytes; + data.linkData.linkLow = 0; + data.linkData.linkHigh = 0; + + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + } +#endif + + /* Wait until MMU configure finishes. */ + do + { + gckOS_Delay(hardware->os, delay); + + gcmkONERROR(gckOS_ReadRegisterEx( + hardware->os, + hardware->core, + 0x00004, + &idle)); + + timer += delay; + delay *= 2; + +#if gcdGPU_TIMEOUT + if (timer >= hardware->kernel->timeOut) + { + gckHARDWARE_DumpGPUState(hardware); + + if (hardware->kernel->command) + { + gckCOMMAND_DumpExecutingBuffer(hardware->kernel->command); + } + + /* Even if hardware is not reset correctly, let software + ** continue to avoid software stuck. Software will timeout again + ** and try to recover GPU in next timeout. + */ + gcmkONERROR(gcvSTATUS_DEVICE); + } +#endif + } + while (!(((((gctUINT32) (idle)) >> (0 ? 0:0)) & ((gctUINT32) ((((1 ? 0:0) - (0 ? 0:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1)))))) )); + + gcmkDUMP(hardware->os, "@[register.wait 0x%05X 0x%08X 0x%08X]", + 0x00004, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) ((gctUINT32) (~0U) & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))), + idle); + +OnError: + return status; +} + +static gceSTATUS +_FuncValidate_Flush(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + Execution->valid = gcvTRUE; + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FuncRelease_Flush(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + if (Execution->funcVidMem) + { + if (Execution->address) + { + /* Synchroneous unlock. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->funcVidMem, + 0, + gcvNULL + )); + Execution->address = 0; + } + + if (Execution->logical) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->funcVidMem, + 0, + gcvFALSE, + gcvFALSE + )); + Execution->logical = gcvNULL; + } + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->funcVidMem + )); + + Execution->funcVidMem = gcvNULL; + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FuncInit_Flush(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 flushBytes = 0; + gctUINT32 offset = 0; + gctUINT32 endBytes = 0; + gctUINT32 address; + gctUINT32 allocFlag = 0; + gcePOOL pool; + gctUINT8_PTR logical; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + +#if !gcdCAPTURE_ONLY_MODE + pool = gcvPOOL_DEFAULT; +#else + pool = gcvPOOL_VIRTUAL; +#endif + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag = gcvALLOC_FLAG_CACHEABLE; +#endif + + Execution->funcVidMemBytes = 1024; + /* Allocate video memory node for aux functions. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + hardware->kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + allocFlag, + &Execution->funcVidMemBytes, + &pool, + &Execution->funcVidMem + )); + + /* Lock for GPU access. */ + gcmkONERROR(gckVIDMEM_NODE_Lock( + hardware->kernel, + Execution->funcVidMem, + &Execution->address + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + hardware->kernel, + Execution->funcVidMem, + gcvFALSE, + gcvFALSE, + &Execution->logical + )); + /* + ** All cache flush command sequence. + */ + logical = (gctUINT8_PTR)Execution->logical; + address = Execution->address; + + /* Get the size of the flush command. */ + gcmkONERROR(gckHARDWARE_Flush(hardware, gcvFLUSH_ALL, gcvNULL, &flushBytes)); + + /* Append a flush. */ + gcmkONERROR(gckHARDWARE_Flush(hardware, gcvFLUSH_ALL, logical, &flushBytes)); + + offset += flushBytes; + logical += offset; + address += offset; + + if (hardware->wlFE) + { + gcmkONERROR(gckWLFE_End(hardware, gcvNULL, ~0U, &endBytes)); + gcmkONERROR(gckWLFE_End(hardware, logical, address, &endBytes)); + } + + Execution->bytes = flushBytes + endBytes; + Execution->endAddress = Execution->address + flushBytes; + Execution->endLogical = (gctUINT8_PTR)Execution->logical + flushBytes; + + return gcvSTATUS_OK; +OnError: + _FuncRelease_Flush(Execution); + + return status; +} + +#if gcdINITIALIZE_PPU +static gceSTATUS +_FuncValidate_PPU(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + Execution->valid = gcvFALSE; + if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_EVIS2_FLOP_RESET_FIX)) + { + if (hardware->identity.customerID == 0x21 || + hardware->identity.customerID == 0x25 || + hardware->identity.customerID == 0x86 + ) + { + Execution->valid = gcvTRUE; + } + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FuncRelease_PPU(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + gctUINT32 i; + + if (Execution->data) + { + for (i = 0; i < 8; i++) + { + if (Execution->data[i].bufVidMem) + { + if (Execution->data[i].logical) + { + gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvFALSE, + gcvFALSE + ); + Execution->data[i].logical = gcvNULL; + } + if (Execution->data[i].address) + { + gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvNULL + ); + Execution->data[i].address = 0; + } + + gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->data[i].bufVidMem + ); + } + } + + gcmkVERIFY_OK(gckOS_Free(hardware->os, Execution->data)); + Execution->data = gcvNULL; + } + + if (Execution->funcVidMem) + { + if (Execution->address) + { + /* Synchroneous unlock. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->funcVidMem, + 0, + gcvNULL + )); + Execution->address = 0; + } + + if (Execution->logical) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->funcVidMem, + 0, + gcvFALSE, + gcvFALSE + )); + Execution->logical = gcvNULL; + } + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->funcVidMem + )); + + Execution->funcVidMem = gcvNULL; + } + + return gcvSTATUS_OK; +} + +gceSTATUS +_PatchPPUBuffer( + IN gcsFUNCTION_EXECUTION_PTR Execution + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gcePOOL pool = gcvPOOL_DEFAULT; + gctUINT32_PTR command = (gctUINT32_PTR)Execution->logical; + gctUINT32 commandPatchOffsets[8]; + gctUINT32_PTR patchBuffers[8]; + gctSIZE_T patchBufferSizes[8]; + gctUINT32 i; + gctUINT32 allocFlag = 0; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + gctPOINTER pointer = gcvNULL; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag = gcvALLOC_FLAG_CACHEABLE; +#endif + + patchBuffers[0] = ppuMem0; + patchBuffers[1] = ppuMem1; + patchBuffers[2] = ppuMem2; + patchBuffers[3] = ppuMem3; + patchBuffers[4] = ppuMem4; + patchBuffers[5] = ppuMem5; + patchBuffers[6] = ppuMem6; + patchBuffers[7] = ppuMem7; + + patchBufferSizes[0] = gcmSIZEOF(ppuMem0); + patchBufferSizes[1] = gcmSIZEOF(ppuMem1); + patchBufferSizes[2] = gcmSIZEOF(ppuMem2); + patchBufferSizes[3] = gcmSIZEOF(ppuMem3); + patchBufferSizes[4] = gcmSIZEOF(ppuMem4); + patchBufferSizes[5] = gcmSIZEOF(ppuMem5); + patchBufferSizes[6] = gcmSIZEOF(ppuMem6); + patchBufferSizes[7] = gcmSIZEOF(ppuMem7); + + commandPatchOffsets[0] = PATCH_OFFSET_0; + commandPatchOffsets[1] = PATCH_OFFSET_1; + commandPatchOffsets[2] = PATCH_OFFSET_2; + commandPatchOffsets[4] = PATCH_OFFSET_4; + commandPatchOffsets[5] = PATCH_OFFSET_5; + commandPatchOffsets[6] = PATCH_OFFSET_6; + commandPatchOffsets[7] = PATCH_OFFSET_7; + + gcmkONERROR(gckOS_Allocate(hardware->os, + gcmSIZEOF(gcsFUNCTION_EXECUTION_DATA) * 8, + &pointer)); + + gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsFUNCTION_EXECUTION_DATA) * 8); + + Execution->data = (gcsFUNCTION_EXECUTION_DATA_PTR)pointer; + + for (i = 0; i < 8; i++) + { + if (i != 3) + { + /* Allocate memory for buffers. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + hardware->kernel, + 64, + gcvVIDMEM_TYPE_BITMAP, + allocFlag, + &patchBufferSizes[i], + &pool, + &Execution->data[i].bufVidMem + )); + + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + hardware->kernel, + Execution->data[i].bufVidMem, + gcvFALSE, + gcvFALSE, + &Execution->data[i].logical + )); + + gckOS_MemCopy(Execution->data[i].logical, patchBuffers[i], patchBufferSizes[i]); + + gcmkONERROR(gckVIDMEM_NODE_Lock( + hardware->kernel, + Execution->data[i].bufVidMem, + &Execution->data[i].address + )); + + /* patch the buffer */ + command[commandPatchOffsets[i]] = Execution->data[i].address; + } + } + + return status; +OnError: + if (Execution->data) + { + for (i = 0; i < 8; i++) + { + if (Execution->data[i].bufVidMem) + { + if (Execution->data[i].logical) + { + gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvFALSE, + gcvFALSE + ); + Execution->data[i].logical = gcvNULL; + } + if (Execution->data[i].address) + { + gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvNULL + ); + Execution->data[i].address = 0; + } + + gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->data[i].bufVidMem + ); + } + } + + gcmkVERIFY_OK(gckOS_Free(hardware->os, Execution->data)); + Execution->data = gcvNULL; + } + + return status; +} + +gceSTATUS +_InitializePPU( + IN gcsFUNCTION_EXECUTION_PTR Execution, + IN OUT gctUINT32 * Bytes + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctPOINTER logical = Execution->logical; + + gctUINT32 flushCommands[] = { + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x028A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000011, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E13) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000002, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x5580) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000002, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000701, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + 0x00000701, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0xD8000340, 0x00000080, 0x00800080, 0x444051F0, 0xFFFFFFFF, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800 + 0x08) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0xD8004380, 0x00000040, 0x00400040, 0x444051F0, 0xFFFFFFFF, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800 + 0x10) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0xD80053C0, 0x00000040, 0x00400040, 0x444051F0, 0xFFFFFFFF, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800 + 0x18) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0xD8000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800 + 0x19) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0xD8000040, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800 + 0x1C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (8) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0xD8006400, 0x00000080, 0x00400040, 0x44405071, 0x00000003, 0x00002000, 0x00000000, 0x00000000, 0xFFFFFFFF, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800 + 0x24) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x11111111, 0x00000000, 0x03020100, 0x07060504, 0xFFFFFFFF, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800 + 0x28) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x22222222, 0x00000000, 0x00000000, 0x00000400, 0xFFFFFFFF, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800 + 0x2C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000001, 0x00000001, 0x00000001, 0x00000001, 0xFFFFFFFF, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800 + 0x30) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (4) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000001, 0x00000001, 0x00000001, 0x00000001, 0xFFFFFFFF, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x022C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x0000001F, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x02A4) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0420) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0403) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000005, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0416) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0409) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x021F) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0424) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x0000000F, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x040A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0xD8000100, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x5580) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000002, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x021A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000001, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0425) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x0000000E, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0402) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00001F01, 0x00000005, 0x00000F00, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0228) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x02AA) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x028C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000100, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E07) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x040C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x020C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x0F3F0000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0201) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000001, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E22) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0412) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0240) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x03000002, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0249) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0247) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000001, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x024B) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x024D) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x024F) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0256) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000008, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0257) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000001, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0258) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0250) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (6) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, 0x0000003F, 0xFFFFFFFF, 0x00000007, 0x00000000, 0x000003FF, 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0248) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0xBADABEEB, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000C20, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E02) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000701, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x09 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + 0x00000701, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000C23, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000C23, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0594) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000001, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000C23, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E13) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))), + 0x00000000, + + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x02 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))), + 0x00000000, + }; + + *Bytes = gcmSIZEOF(flushCommands) - 8; + /* Patch the command. */ + if (logical) + { + gckOS_MemCopy(logical, flushCommands, *Bytes); + status = _PatchPPUBuffer(Execution); + } + + return status; +} + +static gceSTATUS +_FuncInit_PPU(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 ppuBytes = 0; + gctUINT32 offset = 0; + gctUINT32 endBytes = 0; + gctUINT32 address; + gctUINT32 allocFlag = 0; + gcePOOL pool; + gctUINT8_PTR logical; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + pool = gcvPOOL_DEFAULT; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag = gcvALLOC_FLAG_CACHEABLE; +#endif + Execution->funcVidMemBytes = 1024; + /* Allocate video memory node for aux functions. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + hardware->kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + allocFlag, + &Execution->funcVidMemBytes, + &pool, + &Execution->funcVidMem + )); + + /* Lock for GPU access. */ + gcmkONERROR(gckVIDMEM_NODE_Lock( + hardware->kernel, + Execution->funcVidMem, + &Execution->address + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + hardware->kernel, + Execution->funcVidMem, + gcvFALSE, + gcvFALSE, + &Execution->logical + )); + /* + ** All cache flush command sequence. + */ + logical = (gctUINT8_PTR)Execution->logical; + address = Execution->address; + /* Append a flush. */ + gcmkONERROR(_InitializePPU(Execution, &ppuBytes)); + + offset += ppuBytes; + logical += offset; + address += offset; + + if (hardware->wlFE) + { + gcmkONERROR(gckWLFE_End(hardware, gcvNULL, ~0U, &endBytes)); + gcmkONERROR(gckWLFE_End(hardware, logical, address, &endBytes)); + } + + Execution->bytes = ppuBytes + endBytes; + Execution->endAddress = Execution->address + ppuBytes; + Execution->endLogical = (gctUINT8_PTR)Execution->logical + ppuBytes; + + return gcvSTATUS_OK; +OnError: + _FuncRelease_PPU(Execution); + + return status; +} +#endif + +#if gcdRESET_USC +gceSTATUS +_InitializeUSC( + IN gcsFUNCTION_EXECUTION_PTR Execution, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT32 i, idx = 0; + gcePOOL pool = gcvPOOL_DEFAULT; + gctUINT32 flushCommands[128] = {0}; + gctUINT32_PTR nnCommands = gcvNULL; + gctSIZE_T patchBufferSizes[4]; + gceSTATUS status = gcvSTATUS_OK; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + gctUINT8_PTR logical = (gctUINT8_PTR)Execution->logical; + gctPOINTER pointer = gcvNULL; + + if (hardware->identity.customerID == 0x86) + { + patchBufferSizes[NN_BUFFER_IDX] = gcmSIZEOF(nnBuffer0); + nnCommands = nnBuffer0; + } + else if (hardware->identity.customerID == 0x25) + { + patchBufferSizes[NN_BUFFER_IDX] = gcmSIZEOF(nnBuffer1); + nnCommands = nnBuffer1; + } + else if (hardware->identity.customerID == 0x21) + { + patchBufferSizes[NN_BUFFER_IDX] = gcmSIZEOF(nnBuffer2); + nnCommands = nnBuffer2; + } + else + { + gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); + } + + patchBufferSizes[INPUT_BUFFER_IDX] = + patchBufferSizes[KERNEL_BUFFER_IDX] = + patchBufferSizes[RESULT_BUFFER_IDX] = 1024 * 1024; + + gcmkONERROR(gckOS_Allocate(hardware->os, + gcmSIZEOF(gcsFUNCTION_EXECUTION_DATA) * 4, + &pointer)); + + gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsFUNCTION_EXECUTION_DATA) * 4); + + Execution->data = (gcsFUNCTION_EXECUTION_DATA_PTR)pointer; + + for (i = 0; i < 4; i++) + { + /* Allocate memory for buffers. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + hardware->kernel, + 64, + gcvVIDMEM_TYPE_BITMAP, + gcvALLOC_FLAG_NONE, + &patchBufferSizes[i], + &pool, + &Execution->data[i].bufVidMem + )); + + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + hardware->kernel, + Execution->data[i].bufVidMem, + gcvFALSE, + gcvFALSE, + &Execution->data[i].logical + )); + + gcmkONERROR(gckVIDMEM_NODE_Lock( + hardware->kernel, + Execution->data[i].bufVidMem, + &Execution->data[i].address + )); + + gcmkONERROR(gckOS_ZeroMemory(Execution->data[i].logical, patchBufferSizes[i])); + } + + /* Patch NN command */ + nnCommands[PATCH_KERNEL_OFFSET] = Execution->data[KERNEL_BUFFER_IDX].address >> 6; + nnCommands[PATCH_INPUT_OFFSET] = Execution->data[INPUT_BUFFER_IDX].address; + nnCommands[PATCH_RESULT_OFFSET] = Execution->data[RESULT_BUFFER_IDX].address; + gckOS_MemCopy(Execution->data[NN_BUFFER_IDX].logical, nnCommands, patchBufferSizes[NN_BUFFER_IDX]); + + /* construct command */ + if (hardware->identity.customerID == 0x21) + { + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x006B) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000040; + } + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0529) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0xfffffe7f; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0500) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x04011311; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0501) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0502) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x3f800000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0503) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x4b7fffff; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0504) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + if (hardware->identity.customerID == 0x21) + { + flushCommands[idx++] = 0x000a0000; + } + else + { + flushCommands[idx++] = 0x00000000; + } + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0520) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + if (hardware->identity.customerID == 0x21) + { + flushCommands[idx++] = 0x000a0000; + } + else + { + flushCommands[idx++] = 0x00000000; + } + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0505) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000500; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0506) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00070007; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x050B) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x05102f0a; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E06) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x050C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0518) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x050D) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000500; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0300) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0301) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0302) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x43a00000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0303) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x43700000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0308) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x43a07800; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0309) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x4370f000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0595) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00400000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0380) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000004; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0515) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0516) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0518) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0520) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + if (hardware->identity.customerID == 0x21) + { + flushCommands[idx++] = 0x000a0000; + } + else + { + flushCommands[idx++] = 0x00000000; + } + + if (hardware->identity.customerID == 0x86) + { + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E4C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x0000008c; + } + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0428) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = Execution->data[NN_BUFFER_IDX].address; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000040; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x02 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + flushCommands[idx++] = 0x00000000; + + *Bytes = idx * 4 - 8; + /* Copy command to command buffer */ + gckOS_MemCopy(logical, flushCommands, *Bytes); + + return status; + +OnError: + if (Execution->data) + { + for (i = 0; i < 4; i++) + { + if (Execution->data[i].bufVidMem) + { + if (Execution->data[i].logical) + { + gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvFALSE, + gcvFALSE + ); + Execution->data[i].logical = gcvNULL; + } + if (Execution->data[i].address) + { + gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvNULL + ); + Execution->data[i].address = 0; + } + + gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->data[i].bufVidMem + ); + } + } + + gcmkVERIFY_OK(gckOS_Free(hardware->os, Execution->data)); + Execution->data = gcvNULL; + } + + return status; +} + +static gceSTATUS +_FuncValidate_USC(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + Execution->valid = gcvFALSE; + if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_ASYNC_CP_RTN_FLOP_RESET_FIX)) + { + if (hardware->identity.customerID == 0x21 || + hardware->identity.customerID == 0x25 || + hardware->identity.customerID == 0x86 + ) + { + Execution->valid = gcvTRUE; + } + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FuncRelease_USC(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + gctUINT32 i; + + if (Execution->data) + { + for (i = 0; i < 4; i++) + { + if (Execution->data[i].bufVidMem) + { + if (Execution->data[i].logical) + { + gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvFALSE, + gcvFALSE + ); + Execution->data[i].logical = gcvNULL; + } + if (Execution->data[i].address) + { + gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvNULL + ); + Execution->data[i].address = 0; + } + + gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->data[i].bufVidMem + ); + } + } + + gcmkVERIFY_OK(gckOS_Free(hardware->os, Execution->data)); + Execution->data = gcvNULL; + } + + if (Execution->funcVidMem) + { + if (Execution->address) + { + /* Synchroneous unlock. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->funcVidMem, + 0, + gcvNULL + )); + Execution->address = 0; + } + + if (Execution->logical) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->funcVidMem, + 0, + gcvFALSE, + gcvFALSE + )); + Execution->logical = gcvNULL; + } + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->funcVidMem + )); + + Execution->funcVidMem = gcvNULL; + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FuncInit_USC(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 uscBytes = 0; + gctUINT32 offset = 0; + gctUINT32 endBytes = 0; + gctUINT32 address; + gctUINT32 allocFlag = 0; + gcePOOL pool; + gctPOINTER logical; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + pool = gcvPOOL_DEFAULT; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag = gcvALLOC_FLAG_CACHEABLE; +#endif + + Execution->funcVidMemBytes = 1024; + /* Allocate video memory node for aux functions. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + hardware->kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + allocFlag, + &Execution->funcVidMemBytes, + &pool, + &Execution->funcVidMem + )); + + /* Lock for GPU access. */ + gcmkONERROR(gckVIDMEM_NODE_Lock( + hardware->kernel, + Execution->funcVidMem, + &Execution->address + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + hardware->kernel, + Execution->funcVidMem, + gcvFALSE, + gcvFALSE, + &Execution->logical + )); + /* + ** All cache flush command sequence. + */ + logical = Execution->logical; + address = Execution->address; + /* Append a flush. */ + gcmkONERROR(_InitializeUSC(Execution, &uscBytes)); + + offset += uscBytes; + address += offset; + logical = (gctUINT8_PTR)logical + offset; + + if (hardware->wlFE) + { + gcmkONERROR(gckWLFE_End(hardware, gcvNULL, ~0U, &endBytes)); + gcmkONERROR(gckWLFE_End(hardware, logical, address, &endBytes)); + } + + Execution->bytes = uscBytes + endBytes; + Execution->endAddress = Execution->address + uscBytes; + Execution->endLogical = (gctUINT8_PTR)Execution->logical + uscBytes; + + return gcvSTATUS_OK; +OnError: + _FuncRelease_USC(Execution); + + return status; +} + +gceSTATUS +_InitializeUSC2( + IN gcsFUNCTION_EXECUTION_PTR Execution, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT32 i, idx = 0; + gcePOOL pool = gcvPOOL_DEFAULT; + gctUINT32 flushCommands[80] = {0}; + gctSIZE_T patchBufferSizes[2]; + gceSTATUS status = gcvSTATUS_OK; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + gctUINT8_PTR logical = (gctUINT8_PTR)Execution->logical; + gctPOINTER pointer = gcvNULL; + + patchBufferSizes[INST_BUFFER_IDX] = gcmSIZEOF(instBuffer); + patchBufferSizes[OUTPUT_BUFFER_IDX] = 1024; + + gcmkONERROR(gckOS_Allocate(hardware->os, + gcmSIZEOF(gcsFUNCTION_EXECUTION_DATA) * 2, + &pointer)); + + gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsFUNCTION_EXECUTION_DATA) * 2); + + Execution->data = (gcsFUNCTION_EXECUTION_DATA_PTR)pointer; + + for (i = 0; i < 2; i++) + { + /* Allocate memory for buffers. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + hardware->kernel, + 64, + gcvVIDMEM_TYPE_BITMAP, + gcvALLOC_FLAG_NONE, + &patchBufferSizes[i], + &pool, + &Execution->data[i].bufVidMem + )); + + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + hardware->kernel, + Execution->data[i].bufVidMem, + gcvFALSE, + gcvFALSE, + &Execution->data[i].logical + )); + + gcmkONERROR(gckVIDMEM_NODE_Lock( + hardware->kernel, + Execution->data[i].bufVidMem, + &Execution->data[i].address + )); + + gcmkONERROR(gckOS_ZeroMemory(Execution->data[i].logical, patchBufferSizes[i])); + } + + gckOS_MemCopy(Execution->data[INST_BUFFER_IDX].logical, instBuffer, patchBufferSizes[INST_BUFFER_IDX]); + + /* construct command */ + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x0D & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | gcvCORE_3D_ALL_MASK; + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0xD800) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (0x14) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + flushCommands[idx++] = 0x00000001; + flushCommands[idx++] = 0x00000002; + flushCommands[idx++] = 0x075bcd15; + flushCommands[idx++] = 0x00001000; + flushCommands[idx++] = 0x00010000; + flushCommands[idx++] = 0x00020000; + flushCommands[idx++] = 0x00030000; + flushCommands[idx++] = 0x00000000; + flushCommands[idx++] = 0x00000001; + flushCommands[idx++] = 0x00000002; + flushCommands[idx++] = 0x00000004; + flushCommands[idx++] = 0x0000000c; + flushCommands[idx++] = 0x00000003; + flushCommands[idx++] = 0x00000003; + flushCommands[idx++] = 0x0000000f; + flushCommands[idx++] = Execution->data[OUTPUT_BUFFER_IDX].address; + flushCommands[idx++] = 0x00000000; + flushCommands[idx++] = 0x00000080; + flushCommands[idx++] = 0x00000001; + flushCommands[idx++] = 0xffffffff; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x0D & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | gcvCORE_3D_ALL_MASK; + flushCommands[idx++] = 0xf7f38c3c; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x021F) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0424) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x0000000e; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0402) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000001; + flushCommands[idx++] = 0x00000003; + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x040A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = Execution->data[INST_BUFFER_IDX].address; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0409) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x022C) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000010; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x021A) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000001; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x0D & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | gcvCORE_3D_ALL_MASK; + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E21) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x022E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000300; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0240) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x02000001; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0247) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000001; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0249) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0250) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + flushCommands[idx++] = 0x00000000; + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0253) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + flushCommands[idx++] = 0x00000000; + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x024B) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x024D) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000000; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0256) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (3) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000001; + flushCommands[idx++] = 0x00000001; + flushCommands[idx++] = 0x00000001; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E01) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000020; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0248) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000001; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x00000020; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E01) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + flushCommands[idx++] = 0x0000004f; + + flushCommands[idx++] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x02 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + flushCommands[idx++] = 0x00000000; + + *Bytes = idx * 4 - 8; + /* Copy command to command buffer */ + gckOS_MemCopy(logical, flushCommands, *Bytes); + + return status; +OnError: + if (Execution->data) + { + for (i = 0; i < 2; i++) + { + if (Execution->data[i].bufVidMem) + { + if (Execution->data[i].logical) + { + gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvFALSE, + gcvFALSE + ); + Execution->data[i].logical = gcvNULL; + } + if (Execution->data[i].address) + { + gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvNULL + ); + Execution->data[i].address = 0; + } + + gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->data[i].bufVidMem + ); + } + } + + gcmkVERIFY_OK(gckOS_Free(hardware->os, Execution->data)); + Execution->data = gcvNULL; + } + + return status; +} + +static gceSTATUS +_FuncValidate_USC2(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + Execution->valid = gcvFALSE; + if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_EVICT_CTRL_FIFO_FLOP_RESET_FIX)) + { + if (hardware->identity.customerID == 0x21 || + hardware->identity.customerID == 0x25 || + hardware->identity.customerID == 0x86 + ) + { + Execution->valid = gcvTRUE; + } + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FuncRelease_USC2(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + gctUINT32 i; + + if (Execution->data) + { + for (i = 0; i < 2; i++) + { + if (Execution->data[i].bufVidMem) + { + if (Execution->data[i].logical) + { + gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvFALSE, + gcvFALSE + ); + Execution->data[i].logical = gcvNULL; + } + if (Execution->data[i].address) + { + gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->data[i].bufVidMem, + 0, + gcvNULL + ); + Execution->data[i].address = 0; + } + + gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->data[i].bufVidMem + ); + } + } + + gcmkVERIFY_OK(gckOS_Free(hardware->os, Execution->data)); + Execution->data = gcvNULL; + } + + if (Execution->funcVidMem) + { + if (Execution->address) + { + /* Synchroneous unlock. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( + hardware->kernel, + Execution->funcVidMem, + 0, + gcvNULL + )); + Execution->address = 0; + } + + if (Execution->logical) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + hardware->kernel, + Execution->funcVidMem, + 0, + gcvFALSE, + gcvFALSE + )); + Execution->logical = gcvNULL; + } + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + hardware->kernel, + Execution->funcVidMem + )); + + Execution->funcVidMem = gcvNULL; + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FuncInit_USC2(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 uscBytes = 0; + gctUINT32 offset = 0; + gctUINT32 endBytes = 0; + gctUINT32 address; + gctUINT32 allocFlag = 0; + gcePOOL pool; + gctPOINTER logical; + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + pool = gcvPOOL_DEFAULT; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag = gcvALLOC_FLAG_CACHEABLE; +#endif + + Execution->funcVidMemBytes = 1024; + /* Allocate video memory node for aux functions. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + hardware->kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + allocFlag, + &Execution->funcVidMemBytes, + &pool, + &Execution->funcVidMem + )); + + /* Lock for GPU access. */ + gcmkONERROR(gckVIDMEM_NODE_Lock( + hardware->kernel, + Execution->funcVidMem, + &Execution->address + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + hardware->kernel, + Execution->funcVidMem, + gcvFALSE, + gcvFALSE, + &Execution->logical + )); + /* + ** All cache flush command sequence. + */ + logical = Execution->logical; + address = Execution->address; + /* Append a flush. */ + gcmkONERROR(_InitializeUSC2(Execution, &uscBytes)); + + offset += uscBytes; + address += offset; + logical = (gctUINT8_PTR)logical + offset; + + if (hardware->wlFE) + { + gcmkONERROR(gckWLFE_End(hardware, gcvNULL, ~0U, &endBytes)); + gcmkONERROR(gckWLFE_End(hardware, logical, address, &endBytes)); + } + + Execution->bytes = uscBytes + endBytes; + Execution->endAddress = Execution->address + uscBytes; + Execution->endLogical = (gctUINT8_PTR)Execution->logical + uscBytes; + + return gcvSTATUS_OK; + +OnError: + _FuncRelease_USC2(Execution); + + return status; +} + +#endif + +/******************************************************************************* +** +** gckFUNCTION_Construct +** +** Generate command buffer snippets which will be used by gckHARDWARE, by which +** gckHARDWARE can manipulate GPU by FE command without using gckCOMMAND to avoid +** race condition and deadlock. +** +** Notice: +** 1. Each snippet can only be executed when GPU is idle. +** 2. Execution is triggered by AHB (0x658) +** 3. Each snippet followed by END so software can sync with GPU by checking GPU +** idle +** 4. It is transparent to gckCOMMAND command buffer. +** +** Existing Snippets: +** 1. MMU Configure +** For new MMU, after GPU is reset, FE execute this command sequence to enable MMU. +*/ +gceSTATUS gckFUNCTION_Construct(IN gctPOINTER Hardware) +{ + gceSTATUS status = gcvSTATUS_OK; + gckHARDWARE hardware = (gckHARDWARE)Hardware; + gctPOINTER pointer = gcvNULL; + gctUINT i; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(Hardware != gcvNULL); + + /* Allocate the gcsFUNCTION_EXECUTION object. */ + gcmkONERROR(gckOS_Allocate(hardware->os, + gcmSIZEOF(gcsFUNCTION_EXECUTION) * gcvFUNCTION_EXECUTION_NUM, + &pointer)); + + gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsFUNCTION_EXECUTION) * gcvFUNCTION_EXECUTION_NUM); + + hardware->functions = (gcsFUNCTION_EXECUTION_PTR)pointer; + + for (i = 0; i < gcvFUNCTION_EXECUTION_NUM; i++) + { + gcsFUNCTION_EXECUTION_PTR func = &hardware->functions[i]; + + func->hardware = hardware; + func->funcId = (gceFUNCTION_EXECUTION)i; + + /* Init functions API pointer */ + switch (i) + { + case gcvFUNCTION_EXECUTION_MMU: + gckOS_MemCopy(func->funcName, "set mmu", 8); + func->funcExecution.init = _FuncInit_MMU; + func->funcExecution.validate = _FuncValidate_MMU; + func->funcExecution.execute = _FuncExecute_MMU; + func->funcExecution.release = _FuncRelease_MMU; + break; + + case gcvFUNCTION_EXECUTION_FLUSH: + gckOS_MemCopy(func->funcName, "flush", 6); + func->funcExecution.init = _FuncInit_Flush; + func->funcExecution.validate = _FuncValidate_Flush; + func->funcExecution.execute = _FuncExecute; + func->funcExecution.release = _FuncRelease_Flush; + break; + +#if gcdINITIALIZE_PPU + case gcvFUNCTION_EXECUTION_INITIALIZE_PPU: + gckOS_MemCopy(func->funcName, "init ppu", 9); + func->funcExecution.init = _FuncInit_PPU; + func->funcExecution.validate = _FuncValidate_PPU; + func->funcExecution.execute = _FuncExecute; + func->funcExecution.release = _FuncRelease_PPU; + break; +#endif +#if gcdRESET_USC + case gcvFUNCTION_EXECUTION_RESET_USC: + gckOS_MemCopy(func->funcName, "reset usc", 10); + func->funcExecution.init = _FuncInit_USC; + func->funcExecution.validate = _FuncValidate_USC; + func->funcExecution.execute = _FuncExecute; + func->funcExecution.release = _FuncRelease_USC; + break; + case gcvFUNCTION_EXECUTION_RESET_USC2: + gckOS_MemCopy(func->funcName, "reset usc2", 11); + func->funcExecution.init = _FuncInit_USC2; + func->funcExecution.validate = _FuncValidate_USC2; + func->funcExecution.execute = _FuncExecute; + func->funcExecution.release = _FuncRelease_USC2; + break; +#endif + + default: + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + } + + return gcvSTATUS_OK; +OnError: + if (pointer) + { + gcmkVERIFY_OK(gckOS_Free(hardware->os, pointer)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} +gceSTATUS gckFUNCTION_Destory(IN gctPOINTER Hardware) +{ + gceSTATUS status = gcvSTATUS_OK; + gckHARDWARE hardware = (gckHARDWARE)Hardware; + gctUINT i; + + gcmkHEADER_ARG("Hardware=0x%x", Hardware); + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(Hardware != gcvNULL); + + for (i = 0; i < gcvFUNCTION_EXECUTION_NUM; i++) + { + gckFUNCTION_Release(&hardware->functions[i]); + } + + gcmkVERIFY_OK(gckOS_Free(hardware->os, hardware->functions)); + hardware->functions = gcvNULL; + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS gckFUNCTION_Validate(IN gcsFUNCTION_EXECUTION_PTR Execution, + IN OUT gctBOOL_PTR Valid) +{ + gceSTATUS status = gcvSTATUS_NOT_SUPPORTED; + + gcmkHEADER_ARG("Execution=0x%x", Execution); + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(Execution != gcvNULL); + gcmkVERIFY_ARGUMENT(Valid != gcvNULL); + + if (Execution->funcExecution.validate) + { + status = Execution->funcExecution.validate(Execution); + *Valid = Execution->valid; + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS gckFUNCTION_Init(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gceSTATUS status = gcvSTATUS_NOT_SUPPORTED; + + gcmkHEADER_ARG("Execution=0x%x", Execution); + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(Execution != gcvNULL); + + Execution->inited = gcvTRUE; + if (Execution->funcExecution.init) + { + status = Execution->funcExecution.init(Execution); + } + + if (status != gcvSTATUS_OK) + { + Execution->inited = gcvFALSE; + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS gckFUNCTION_Execute(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gceSTATUS status = gcvSTATUS_NOT_SUPPORTED; + + gcmkHEADER_ARG("Execution=0x%x", Execution); + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(Execution != gcvNULL); + + if (Execution->inited && Execution->funcExecution.execute) + { + status = Execution->funcExecution.execute(Execution); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS gckFUNCTION_Release(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ + gceSTATUS status = gcvSTATUS_NOT_SUPPORTED; + + gcmkHEADER_ARG("Execution=0x%x", Execution); + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(Execution != gcvNULL); + + if (Execution->inited && Execution->funcExecution.release) + { + status = Execution->funcExecution.release(Execution); + } + + Execution->inited = gcvFALSE; + + gcmkFOOTER(); + return status; +} + +void +gckFUNCTION_Dump(IN gcsFUNCTION_EXECUTION_PTR Execution) +{ +#if gcdDUMP_IN_KERNEL + gckHARDWARE hardware = (gckHARDWARE)Execution->hardware; + + gcmkDUMP(hardware->os, "#[function: %s]", Execution->funcName); + gcmkDUMP_BUFFER( + hardware->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + Execution->logical, + Execution->address, + Execution->bytes); +#endif +} + +
diff --git a/hal/kernel/arch/gc_hal_kernel_hardware_func.h b/hal/kernel/arch/gc_hal_kernel_hardware_func.h new file mode 100644 index 0000000..043a679 --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_hardware_func.h
@@ -0,0 +1,158 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_kernel_hardware_func_h_ +#define __gc_hal_kernel_hardware_func_h_ +#include "gc_hal.h" +#ifdef __cplusplus +extern "C" { +#endif + +#define gcdINITIALIZE_PPU 0 +#define gcdRESET_USC 0 + +typedef struct _gcsFUNCTION_EXECUTION * gcsFUNCTION_EXECUTION_PTR; + +typedef enum { + gcvFUNCTION_EXECUTION_MMU, + gcvFUNCTION_EXECUTION_FLUSH, +#if gcdRESET_USC + gcvFUNCTION_EXECUTION_RESET_USC, + gcvFUNCTION_EXECUTION_RESET_USC2, +#endif +#if gcdINITIALIZE_PPU + gcvFUNCTION_EXECUTION_INITIALIZE_PPU, +#endif + + gcvFUNCTION_EXECUTION_NUM +} +gceFUNCTION_EXECUTION; + +typedef struct { + gckVIDMEM_NODE bufVidMem; + gctUINT32 address; + /* CPU address of the function. */ + gctPOINTER logical; +} +gcsFUNCTION_EXECUTION_DATA, *gcsFUNCTION_EXECUTION_DATA_PTR; + +typedef struct { + gceSTATUS (*validate)(IN gcsFUNCTION_EXECUTION_PTR Execution); + gceSTATUS (*init)(IN gcsFUNCTION_EXECUTION_PTR Execution); + gceSTATUS (*execute)(IN gcsFUNCTION_EXECUTION_PTR Execution); + gceSTATUS (*release)(IN gcsFUNCTION_EXECUTION_PTR Execution); +} +gcsFUNCTION_API, *gcsFUNCTION_API_PTR; + + +typedef struct _gcsFUNCTION_EXECUTION +{ + gctPOINTER hardware; + + /* Function name */ + gctCHAR funcName[16]; + + /* Function ID */ + gceFUNCTION_EXECUTION funcId; + + /* Function Vidmem object */ + gckVIDMEM_NODE funcVidMem; + + /* Total bytes of the function. */ + gctSIZE_T funcVidMemBytes; + + /* Entry of the function. */ + gctUINT32 address; + + /* CPU address of the function. */ + gctPOINTER logical; + + /* Actually bytes of the function. */ + gctUINT32 bytes; + + /* Hardware address of END in this function. */ + gctUINT32 endAddress; + + /* Logical of END in this function. */ + gctUINT8_PTR endLogical; + + /* Function private data */ + gcsFUNCTION_EXECUTION_DATA_PTR data; + + /* API of functions */ + gcsFUNCTION_API funcExecution; + + /* Need this function or not */ + gctBOOL valid; + + /* Function has inited */ + gctBOOL inited; +} +gcsFUNCTION_EXECUTION; + +gceSTATUS gckFUNCTION_Construct(IN gctPOINTER Hardware); +gceSTATUS gckFUNCTION_Destory(IN gctPOINTER Hardware); + +gceSTATUS gckFUNCTION_Validate(IN gcsFUNCTION_EXECUTION_PTR Execution, + IN OUT gctBOOL_PTR Valid); +gceSTATUS gckFUNCTION_Init(IN gcsFUNCTION_EXECUTION_PTR Execution); +gceSTATUS gckFUNCTION_Execute(IN gcsFUNCTION_EXECUTION_PTR Execution); +gceSTATUS gckFUNCTION_Release(IN gcsFUNCTION_EXECUTION_PTR Execution); +void gckFUNCTION_Dump(IN gcsFUNCTION_EXECUTION_PTR Execution); +#ifdef __cplusplus +} +#endif +#endif +
diff --git a/hal/kernel/arch/gc_hal_kernel_hardware_mc_fe.c b/hal/kernel/arch/gc_hal_kernel_hardware_mc_fe.c new file mode 100644 index 0000000..6d537d0 --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_hardware_mc_fe.c
@@ -0,0 +1,889 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal.h" +#include "gc_hal_kernel.h" +#include "gc_hal_kernel_context.h" + +#define _GC_OBJ_ZONE gcvZONE_HARDWARE + +typedef struct _gcsMCFE_DESCRIPTOR +{ + gctUINT32 start; + gctUINT32 end; +} +gcsMCFE_DESCRIPTOR; + +/* 2^DepthExp = Depth. */ +#define MCFE_RINGBUF_DEPTH_EXP 9 +/* Depth. */ +#define MCFE_RINGBUF_DEPTH (1 << MCFE_RINGBUF_DEPTH_EXP) +/* MCFE descriptor size in bytes, fixed 8. */ +#define MCFE_COMMAND_DESC_SIZE 8 +/* FIFO size in bytes. */ +#define MCFE_RINGBUF_SIZE (MCFE_RINGBUF_DEPTH * MCFE_COMMAND_DESC_SIZE) + +typedef struct _gcsMCFE_RING_BUF +{ + gckVIDMEM_NODE ringBufVideoMem; + gctUINT32 ringBufAddress; + gctUINT32 * ringBufLogical; + gctSIZE_T ringBufBytes; + + gctUINT32 gpuAddress; + gctPHYS_ADDR_T physical; + + /* Read ptr should be often out-of-date. */ + gctUINT32 readPtr; + gctUINT32 writePtr; +} +gcsMCFE_RING_BUF; + +typedef struct _gcsMCFE_CHANNEL +{ + gceMCFE_CHANNEL_TYPE binding; + gcsMCFE_RING_BUF stdRingBuf; + gcsMCFE_RING_BUF priRingBuf; +} +gcsMCFE_CHANNEL; + +struct _gckMCFE +{ + gctUINT32 channelCount; + gctBOOL mmuEnabled; + + /* + * Channels must be the last field. + * Will allocate struct size according to channel count. + */ + gcsMCFE_CHANNEL channels[1]; +}; + +static gcmINLINE gctUINT32 +_NextPtr( + gctUINT32 Ptr + ) +{ + return (Ptr + 1) & (MCFE_RINGBUF_DEPTH - 1); +} + +static gceSTATUS +_AllocateDescRingBuf( + gckHARDWARE Hardware, + gcsMCFE_RING_BUF * Channel + ) +{ + gceSTATUS status; + gcePOOL pool = gcvPOOL_DEFAULT; + gckKERNEL kernel = Hardware->kernel; + gctUINT32 allocFlag = 0; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag |= gcvALLOC_FLAG_CACHEABLE; +#endif + + Channel->ringBufBytes = MCFE_RINGBUF_SIZE; + + /* Allocate video memory node for mcfe ring buffer. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + allocFlag, + &Channel->ringBufBytes, + &pool, + &Channel->ringBufVideoMem + )); + + /* Lock for GPU access. */ + gcmkONERROR(gckVIDMEM_NODE_Lock( + kernel, + Channel->ringBufVideoMem, + &Channel->gpuAddress + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + kernel, + Channel->ringBufVideoMem, + gcvFALSE, + gcvFALSE, + (gctPOINTER *)&Channel->ringBufLogical + )); + + /* Get CPU physical address. */ + gcmkONERROR(gckVIDMEM_NODE_GetPhysical( + kernel, + Channel->ringBufVideoMem, + 0, + &Channel->physical + )); + + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( + Hardware->os, Channel->physical, &Channel->physical)); + + if (Channel->physical > 0xffffffffull) + { + gcmkPRINT("%s(%d): MCFE ring buffer physical over 4G: 0x%llx", + __FUNCTION__, __LINE__, (unsigned long long)Channel->physical); + } + + /* Default to use physical. */ + Channel->ringBufAddress = (gctUINT32)Channel->physical; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +static void +_DestroyDescRingBuf( + gckHARDWARE Hardware, + gcsMCFE_RING_BUF * Channel + ) +{ + gckKERNEL kernel = Hardware->kernel; + + if (Channel->ringBufVideoMem) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + kernel, + Channel->ringBufVideoMem, + 0, + gcvFALSE, + gcvFALSE + )); + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + kernel, + Channel->ringBufVideoMem + )); + + Channel->ringBufVideoMem = gcvNULL; + Channel->ringBufLogical = gcvNULL; + } +} + +static gcmINLINE void +_DestroyMCFE( + IN gckHARDWARE Hardware, + IN gckMCFE FE + ) +{ + if (FE) + { + gctUINT i; + + for (i = 0; i < FE->channelCount; i++) + { + if (FE->channels[i].binding) + { + _DestroyDescRingBuf(Hardware, &FE->channels[i].stdRingBuf); + _DestroyDescRingBuf(Hardware, &FE->channels[i].priRingBuf); + } + } + + gcmkOS_SAFE_FREE(Hardware->os, FE); + } +} + + +static gceSTATUS +_ConstructChannel( + IN gckHARDWARE Hardware, + IN gceMCFE_CHANNEL_TYPE ChannelType, + IN gcsMCFE_CHANNEL * Channel + ) +{ + Channel->binding = ChannelType; + + return gcvSTATUS_OK; +} + +gceSTATUS +gckMCFE_Construct( + IN gckHARDWARE Hardware, + OUT gckMCFE *FE + ) +{ + gceSTATUS status; + gckMCFE fe = gcvNULL; + gctUINT32 i; + gctSIZE_T size = sizeof(struct _gckMCFE); + + if (Hardware->mcfeChannelCount > 1) + { + size += sizeof(gcsMCFE_CHANNEL) * (Hardware->mcfeChannelCount - 1); + } + + gcmkONERROR(gckOS_Allocate(Hardware->os, + size, + (gctPOINTER *)&fe)); + + gckOS_ZeroMemory(fe, size); + + fe->channelCount = Hardware->mcfeChannelCount; + + for (i = 0; i < fe->channelCount; i++) + { + gcmkONERROR( + _ConstructChannel(Hardware, + Hardware->mcfeChannels[i], + &fe->channels[i])); + } + + /* Enable all events. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + 0x00014, + 0xFFFFFFFF)); + + *FE = fe; + return gcvSTATUS_OK; + +OnError: + _DestroyMCFE(Hardware, fe); + return status; +} + +void +gckMCFE_Destroy( + IN gckHARDWARE Hardware, + IN gckMCFE FE + ) +{ + _DestroyMCFE(Hardware, FE); +} + +static gceSTATUS +_ProgramDescRingBuf( + IN gckHARDWARE Hardware, + IN gctBOOL MMUEnabled, + IN gcsMCFE_RING_BUF * Channel, + IN gctUINT32 Index, + IN gctBOOL Priority + ) +{ + gctUINT32 ringBufStartReg; + gctUINT32 depthExpReg; + gctUINT32 readPtrReg; + gctUINT32 writePtrReg; + gctUINT32 data = 0; + + if (Priority) + { + ringBufStartReg = 0x02800; + depthExpReg = 0x02900; + readPtrReg = 0x02B00; + writePtrReg = 0x02A00; + } + else + { + ringBufStartReg = 0x02400; + depthExpReg = 0x02500; + readPtrReg = 0x02700; + writePtrReg = 0x02600; + } + + ringBufStartReg += Index << 2; + depthExpReg += Index << 2; + readPtrReg += Index << 2; + writePtrReg += Index << 2; + + Channel->ringBufAddress = MMUEnabled ? Channel->gpuAddress + : (gctUINT32)Channel->physical; + + /* Channel ringBuf start address. */ + gcmkVERIFY_OK(gckOS_WriteRegisterEx( + Hardware->os, Hardware->core, ringBufStartReg, Channel->ringBufAddress)); + + /* Channel ringBuf depth (exponent of 2). */ + gcmkVERIFY_OK(gckOS_WriteRegisterEx( + Hardware->os, Hardware->core, depthExpReg, MCFE_RINGBUF_DEPTH_EXP)); + + /* The RD ptr could keep unchanged, read and compute WR ptr. */ + gcmkVERIFY_OK(gckOS_ReadRegisterEx(Hardware->os, Hardware->core, readPtrReg, &data)); + + /* Priority ring buffer write ptr. */ + /* gcmkVERIFY_OK(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, writePtrReg, data)); */ + + /* No valid descriptor initially. */ + Channel->readPtr = Channel->writePtr = data; + + return gcvSTATUS_OK; +} + +static gceSTATUS +_InitializeChannel( + IN gckHARDWARE Hardware, + IN gctBOOL MMUEnabled, + IN gcsMCFE_CHANNEL * Channel, + IN gctUINT32 Index + ) +{ + gceSTATUS status; + + /* Allocate ring buffer descriptor memory. */ + if (!Channel->stdRingBuf.ringBufVideoMem) + { + gcmkONERROR(_AllocateDescRingBuf(Hardware, &Channel->stdRingBuf)); + } + + /* No priority channel in system engine. */ + if (!Channel->priRingBuf.ringBufVideoMem && Index != 0) + { + gcmkONERROR(_AllocateDescRingBuf(Hardware, &Channel->priRingBuf)); + } + + gcmkONERROR(_ProgramDescRingBuf(Hardware, MMUEnabled, &Channel->stdRingBuf, Index, gcvFALSE)); + + /* No priority channel in system engine. */ + if (Channel->binding != gcvMCFE_CHANNEL_SYSTEM) + { + gcmkONERROR(_ProgramDescRingBuf(Hardware, MMUEnabled, &Channel->priRingBuf, Index, gcvTRUE)); + } + + return gcvSTATUS_OK; + +OnError: + /* It's OK to leave ringBuf memory not free'd here. */ + return status; +} + +gceSTATUS +gckMCFE_Initialize( + IN gckHARDWARE Hardware, + IN gctBOOL MMUEnabled, + IN gckMCFE FE + ) +{ + gctUINT32 i; + gceSTATUS status; + + for (i = 0; i < FE->channelCount; i++) + { + if (FE->channels[i].binding) + { + gcmkONERROR(_InitializeChannel(Hardware, MMUEnabled, &FE->channels[i], i)); + } + } + + FE->mmuEnabled = MMUEnabled; + return gcvSTATUS_OK; + +OnError: + return status; +} + +gceSTATUS +gckMCFE_Nop( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctSIZE_T * Bytes + ) +{ + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x *Bytes=%lu", + Hardware, Logical, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + + if (Logical != gcvNULL) + { + if (*Bytes < 8) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + /* Append NOP. */ + logical[0] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + logical[1] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, "0x%x: NOP", Logical); + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the NOP command. */ + *Bytes = 8; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + + +gceSTATUS +gckMCFE_Event( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT8 Event, + IN gceKERNEL_WHERE FromWhere, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT size; + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x Event=%u FromWhere=%d *Bytes=%lu", + Hardware, Logical, Event, FromWhere, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + gcmkVERIFY_ARGUMENT(Event < 32); + + /* Ignored. */ + (void)FromWhere; + + size = 8; + + if (Logical != gcvNULL) + { + if (*Bytes < size) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + /* Append EVENT(Event). */ + logical[0] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x16 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) (0x006 & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | Event; + + logical[1] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + { + gctPHYS_ADDR_T phys; + gckOS_GetPhysicalAddress(Hardware->os, Logical, &phys); + gckOS_CPUPhysicalToGPUPhysical(Hardware->os, phys, &phys); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%08x: EVENT %d", phys, Event); + } +#endif + +#if gcdINTERRUPT_STATISTIC + if (Event < gcmCOUNTOF(Hardware->kernel->eventObj->queues)) + { + gckOS_AtomSetMask(Hardware->pendingEvent, 1 << Event); + } +#endif + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the EVENT command. */ + *Bytes = size; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckMCFE_SendSemaphore( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 SemaId, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x SemaId=%u *Bytes=%lu", + Hardware, Logical, SemaId, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + gcmkVERIFY_ARGUMENT(SemaId < 0xFFFF); + + if (Logical != gcvNULL) + { + if (*Bytes < 8) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + /* Append SEND_SEMAPHORE(SemaId). */ + logical[0] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x16 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) (0x002 & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | SemaId; + + logical[1] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + } + + if (Bytes != gcvNULL) + { + *Bytes = 8; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckMCFE_WaitSemaphore( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 SemaId, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x SemaId=%u *Bytes=%lu", + Hardware, Logical, SemaId, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + gcmkVERIFY_ARGUMENT(SemaId < 0xFFFF); + + if (Logical != gcvNULL) + { + if (*Bytes < 8) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + /* Append WAIT_SEMAPHORE(SemaId). */ + logical[0] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x16 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) (0x003 & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | SemaId; + + logical[1] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + } + + if (Bytes != gcvNULL) + { + *Bytes = 8; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckMCFE_Execute( + IN gckHARDWARE Hardware, + IN gctBOOL Priority, + IN gctUINT32 ChannelId, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ) +{ + gctUINT32 regBase; + gcsMCFE_DESCRIPTOR *desc; + gcsMCFE_CHANNEL * channel = &Hardware->mcFE->channels[ChannelId]; + gcsMCFE_RING_BUF * ringBuf = Priority ? &channel->priRingBuf + : &channel->stdRingBuf; + + /* No priority channel in system channel by design. */ + gcmkASSERT(!(channel->binding == gcvMCFE_CHANNEL_SYSTEM && Priority == 1)); + + while (_NextPtr(ringBuf->writePtr) == ringBuf->readPtr) + { + gctUINT32 data; + regBase = Priority ? 0x02B00 + : 0x02700; + + gcmkVERIFY_OK(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + regBase + ChannelId * 4, + &data)); + + ringBuf->readPtr = data; + + if (_NextPtr(ringBuf->writePtr) == ringBuf->readPtr) + { + gcmkPRINT("%s: MCFE channel %s-%d ringBuf is full!", + __FUNCTION__, + Priority ? "Pri" : "Std", + ChannelId); + + gckOS_Delay(Hardware->os, 100); + } + } + + regBase = Priority ? 0x02A00 + : 0x02600; + + /* ringBufLogical is in uint32, 2 uint32 contributes 1 descriptr. */ + desc = (gcsMCFE_DESCRIPTOR *)&ringBuf->ringBufLogical[ringBuf->writePtr * 2]; + desc->start = Address; + desc->end = Address + Bytes; + + gcmkDUMP(Hardware->os, + "#[descriptor %d: channel %s-%d]", + ringBuf->writePtr, + Priority ? "Pri" : "Std", + ChannelId); + + gcmkDUMP_BUFFER(Hardware->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + desc, + ringBuf->ringBufAddress + ringBuf->writePtr * 8, + 8); + + gcmkVERIFY_OK(gckVIDMEM_NODE_CleanCache(Hardware->kernel, + ringBuf->ringBufVideoMem, + 0, + desc, + 8)); + + ringBuf->writePtr = _NextPtr(ringBuf->writePtr); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%08X - 0x%08X: %06d bytes, Channel=%s-%d", + desc->start, desc->end, Bytes, + Priority ? "Pri" : "Std", ChannelId); + + gcmkVERIFY_OK(gckOS_WriteRegisterEx(Hardware->os, + Hardware->core, + regBase + ChannelId * 4, + ringBuf->writePtr)); + + return gcvSTATUS_OK; +} + +gceSTATUS +gckMCFE_HardwareIdle( + IN gckHARDWARE Hardware, + OUT gctBOOL_PTR isIdle + ) +{ + gceSTATUS status; + gctUINT32 idle; + gctUINT32 regRBase; + gctUINT32 readPtr; + gctUINT32 ChannelId = 0; + gctBOOL Priority = gcvFALSE; + gcsMCFE_CHANNEL * channel = &Hardware->mcFE->channels[ChannelId]; + gcsMCFE_RING_BUF * ringBuf = Priority ? &channel->priRingBuf + : &channel->stdRingBuf; + + gcmkHEADER(); + + *isIdle = gcvTRUE; + + /* Read idle register. */ + gcmkONERROR( + gckOS_ReadRegisterEx(Hardware->os, Hardware->core, 0x00004, &idle)); + + /* Pipe must be idle. */ + if ((idle | (1 << 14)) != 0x7fffffff) + { + /* Something is busy. */ + *isIdle = gcvFALSE; + return status; + } + + regRBase = Priority ? 0x02B00 + : 0x02700; + + gcmkONERROR(gckOS_ReadRegisterEx(Hardware->os, + Hardware->core, + regRBase + ChannelId * 4, + &readPtr)); + + if (readPtr != ringBuf->writePtr) + { + *isIdle = gcvFALSE; + } + + gcmkFOOTER(); + +OnError: + return status; +} + +
diff --git a/hal/kernel/arch/gc_hal_kernel_hardware_waitlink_fe.c b/hal/kernel/arch/gc_hal_kernel_hardware_waitlink_fe.c new file mode 100644 index 0000000..ef971ce --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_hardware_waitlink_fe.c
@@ -0,0 +1,1758 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal.h" +#include "gc_hal_kernel.h" +#include "gc_hal_kernel_context.h" + +#define _GC_OBJ_ZONE gcvZONE_HARDWARE + +gceSTATUS +gckWLFE_Construct( + IN gckHARDWARE Hardware, + OUT gckWLFE * FE + ) +{ + /* Just a non-null value. */ + *FE = (gckWLFE)(gctUINTPTR_T)1; + return gcvSTATUS_OK; +} + +void +gckWLFE_Destroy( + IN gckHARDWARE Hardware, + IN gckWLFE FE + ) +{ + gcmkASSERT(FE); +} + +gceSTATUS +gckWLFE_Initialize( + IN gckHARDWARE Hardware, + IN gckWLFE FE + ) +{ + gcmkASSERT(FE); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckWLFE_WaitLink +** +** Append a WAIT/LINK command sequence at the specified location in the command +** queue. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** gctPOINTER Logical +** Pointer to the current location inside the command queue to append +** WAIT/LINK command sequence at or gcvNULL just to query the size of the +** WAIT/LINK command sequence. +** +** gctUINT32 Address +** GPU address of current location inside the command queue. +** +** gctUINT32 Offset +** Offset into command buffer required for alignment. +** +** gctSIZE_T * Bytes +** Pointer to the number of bytes available for the WAIT/LINK command +** sequence. If 'Logical' is gcvNULL, this argument will be ignored. +** +** OUTPUT: +** +** gctSIZE_T * Bytes +** Pointer to a variable that will receive the number of bytes required +** by the WAIT/LINK command sequence. If 'Bytes' is gcvNULL, nothing will +** be returned. +** +** gctUINT32 * WaitOffset +** Pointer to a variable that will receive the offset of the WAIT command +** from the specified logcial pointer. +** If 'WaitOffset' is gcvNULL nothing will be returned. +** +** gctSIZE_T * WaitSize +** Pointer to a variable that will receive the number of bytes used by +** the WAIT command. If 'LinkSize' is gcvNULL nothing will be returned. +*/ +gceSTATUS +gckWLFE_WaitLink( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN gctUINT32 Offset, + IN OUT gctUINT32 * Bytes, + OUT gctUINT32 * WaitOffset, + OUT gctUINT32 * WaitSize + ) +{ + gceSTATUS status; + gctUINT32_PTR logical; + gctUINT32 bytes; + gctBOOL useL2; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x Offset=0x%08x *Bytes=%lu", + Hardware, Logical, Offset, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical != gcvNULL) || (Bytes != gcvNULL)); + + gcmkASSERT(Hardware->wlFE); + useL2 = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_64K_L2_CACHE); + + /* Compute number of bytes required. */ + if (useL2) + { + bytes = gcmALIGN(Offset + 24, 8) - Offset; + } + else + { + bytes = gcmALIGN(Offset + 16, 8) - Offset; + } + + /* Cast the input pointer. */ + logical = (gctUINT32_PTR) Logical; + + if (logical != gcvNULL) + { + /* Not enough space? */ + if (*Bytes < bytes) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + gcmkASSERT(Address != ~0U); + + /* Store the WAIT/LINK address. */ + Hardware->lastWaitLink = Address; + + /* Append WAIT(count). */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x07 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (Hardware->waitCount) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + logical++; + + if (useL2) + { + /* LoadState(AQFlush, 1), flush. */ + *logical++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E03) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:6) - (0 ? + 6:6) + 1))))))) << (0 ? + 6:6))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:6) - (0 ? 6:6) + 1))))))) << (0 ? 6:6))); + } + + /* Append LINK(2, address). */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x08 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (bytes >> 3) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ = Address; + + gcmkTRACE_ZONE( + gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%08x: WAIT %u", Address, Hardware->waitCount + ); + + gcmkTRACE_ZONE( + gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%08x: LINK 0x%08x, #%lu", + Address + 8, Address, bytes + ); + + if (WaitOffset != gcvNULL) + { + /* Return the offset pointer to WAIT command. */ + *WaitOffset = 0; + } + + if (WaitSize != gcvNULL) + { + /* Return number of bytes used by the WAIT command. */ + if (useL2) + { + *WaitSize = 16; + } + else + { + *WaitSize = 8; + } + } + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the WAIT/LINK command + ** sequence. */ + *Bytes = bytes; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu *WaitOffset=0x%x *WaitSize=%lu", + gcmOPT_VALUE(Bytes), gcmOPT_VALUE(WaitOffset), + gcmOPT_VALUE(WaitSize)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckWLFE_InvalidatePipe( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT size; + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + gctBOOL blt = gcvFALSE; + gctBOOL multiCluster = gcvFALSE; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x *Bytes=%lu", + Hardware, Logical, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + + gcmkASSERT(Hardware->wlFE); + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_BLT_ENGINE)) + { + /* Send all event from blt. */ + blt = gcvTRUE; + multiCluster = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_MULTI_CLUSTER); + } + + /* Determine the size of the command. */ + size = Hardware->extraEventStates + ? gcmALIGN(8 + (1 + 5) * 4, 8) /* EVENT + 5 STATES */ + : 8; + + if (blt) + { + size += 16; + if (multiCluster) + size += 8; + } + + /* END. */ + size += 8; + + if (Logical != gcvNULL) + { + if (*Bytes < size) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + if (blt) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + if (multiCluster) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x50CE) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (Hardware->identity.clusterAvailMask & Hardware->options.userClusterMask) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))); + } + } + + /* Append EVENT(Event, PE_SRC). */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E01) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:6) - (0 ? + 6:6) + 1))))))) << (0 ? + 6:6))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:6) - (0 ? 6:6) + 1))))))) << (0 ? 6:6))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) ((gctUINT32) (EVENT_ID_INVALIDATE_PIPE) & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))); + + if (blt) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + { + gctPHYS_ADDR_T phys; + gckOS_GetPhysicalAddress(Hardware->os, Logical, &phys); + gckOS_CPUPhysicalToGPUPhysical(Hardware->os, phys, &phys); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%08x: EVENT %d", phys, EVENT_ID_INVALIDATE_PIPE); + } +#endif + + /* Append the extra states. These are needed for the chips that do not + ** support back-to-back events due to the async interface. The extra + ** states add the necessary delay to ensure that event IDs do not + ** collide. */ + if (Hardware->extraEventStates) + { + *logical++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0100) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (5) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + *logical++ = 0; + *logical++ = 0; + *logical++ = 0; + *logical++ = 0; + *logical++ = 0; + } + +#if gcdINTERRUPT_STATISTIC + if (EVENT_ID_INVALIDATE_PIPE < gcmCOUNTOF(Hardware->kernel->eventObj->queues)) + { + gckOS_AtomSetMask(Hardware->pendingEvent, 1 << EVENT_ID_INVALIDATE_PIPE); + } +#endif + + /* Append END. */ + *logical++ = + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x02 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + /* Record the count of execution which is finised by this END. */ + *logical++ = + Hardware->executeCount; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, "0x%x: END", Logical); + + /* Make sure the CPU writes out the data to memory. */ + gcmkONERROR( + gckOS_MemoryBarrier(Hardware->os, Logical)); + + Hardware->lastEnd = Address + size - 8; + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the EVENT command. */ + *Bytes = size; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +void +gckWLFE_DoneInvalidatePipe( + gckHARDWARE Hardware + ) +{ + gctUINT32 resume; + gctUINT32 bytes; + gctUINT32 idle; + gctUINT32 pageSize = Hardware->kernel->command->pageSize; + + gcmkASSERT(Hardware->wlFE); + + /* Make sure FE is idle. */ + do + { + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x00004, + &idle)); + } + while (idle != 0x7FFFFFFF); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x00664, + &resume)); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + Hardware->os, + Hardware->core, + 0x00664, + &resume)); + + gcmkVERIFY_OK(gckWLFE_WaitLink( + Hardware, + gcvNULL, + ~0U, + resume & (pageSize - 1), + &bytes, + gcvNULL, + gcvNULL + )); + + /* Start Command Parser. */ + gcmkVERIFY_OK(gckWLFE_Execute( + Hardware, + resume, + bytes + )); +} + +/******************************************************************************* +** +** gckWLFE_Link +** +** Append a LINK command at the specified location in the command queue. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** gctPOINTER Logical +** Pointer to the current location inside the command queue to append +** the LINK command at or gcvNULL just to query the size of the LINK +** command. +** +** gctUINT32 FetchAddress +** Hardware address of destination of LINK. +** +** gctSIZE_T FetchSize +** Number of bytes in destination of LINK. +** +** gctSIZE_T * Bytes +** Pointer to the number of bytes available for the LINK command. If +** 'Logical' is gcvNULL, this argument will be ignored. +** +** OUTPUT: +** +** gctSIZE_T * Bytes +** Pointer to a variable that will receive the number of bytes required +** for the LINK command. If 'Bytes' is gcvNULL, nothing will be returned. +*/ +gceSTATUS +gckWLFE_Link( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 FetchAddress, + IN gctUINT32 FetchSize, + IN OUT gctUINT32 * Bytes, + OUT gctUINT32 * Low, + OUT gctUINT32 * High + ) +{ + gceSTATUS status; + gctSIZE_T bytes; + gctUINT32 link; + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x FetchAddress=0x%x FetchSize=%lu " + "*Bytes=%lu", + Hardware, Logical, FetchAddress, FetchSize, + gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + + gcmkASSERT(Hardware->wlFE); + + if (Logical != gcvNULL) + { + if (*Bytes < 8) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + gcmkONERROR( + gckOS_WriteMemory(Hardware->os, logical + 1, FetchAddress)); + + if (High) + { + *High = FetchAddress; + } + + /* Make sure the address got written before the LINK command. */ + gcmkONERROR( + gckOS_MemoryBarrier(Hardware->os, logical + 1)); + + /* Compute number of 64-byte aligned bytes to fetch. */ + bytes = gcmALIGN(FetchAddress + FetchSize, 64) - FetchAddress; + + /* Append LINK(bytes / 8), FetchAddress. */ + link = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x08 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (bytes >> 3) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + gcmkONERROR( + gckOS_WriteMemory(Hardware->os, logical, link)); + + if (Low) + { + *Low = link; + } + + /* Memory barrier. */ + gcmkONERROR( + gckOS_MemoryBarrier(Hardware->os, logical)); + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the LINK command. */ + *Bytes = 8; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckWLFE_End +** +** Append an END command at the specified location in the command queue. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** gctPOINTER Logical +** Pointer to the current location inside the command queue to append +** END command at or gcvNULL just to query the size of the END command. +** +** gctUINT32 Address +** GPU address of current location inside the command queue. +** +** gctSIZE_T * Bytes +** Pointer to the number of bytes available for the END command. If +** 'Logical' is gcvNULL, this argument will be ignored. +** +** OUTPUT: +** +** gctSIZE_T * Bytes +** Pointer to a variable that will receive the number of bytes required +** for the END command. If 'Bytes' is gcvNULL, nothing will be returned. +*/ +gceSTATUS +gckWLFE_End( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gctUINT32 address; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x *Bytes=%lu", + Hardware, Logical, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + + gcmkASSERT(Hardware->wlFE); + + if (Logical != gcvNULL) + { + if (*Bytes < 8) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + /* Append END. */ + logical[0] = + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x02 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + /* Record the count of execution which is finised by this END. */ + logical[1] = + Hardware->executeCount; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, "0x%x: END", Logical); + + /* Make sure the CPU writes out the data to memory. */ + gcmkONERROR( + gckOS_MemoryBarrier(Hardware->os, Logical)); + + gcmkASSERT(Address != ~0U); + address = Address; + + Hardware->lastEnd = address; + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the END command. */ + *Bytes = 8; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckWLFE_Nop +** +** Append a NOP command at the specified location in the command queue. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** gctPOINTER Logical +** Pointer to the current location inside the command queue to append +** NOP command at or gcvNULL just to query the size of the NOP command. +** +** gctSIZE_T * Bytes +** Pointer to the number of bytes available for the NOP command. If +** 'Logical' is gcvNULL, this argument will be ignored. +** +** OUTPUT: +** +** gctSIZE_T * Bytes +** Pointer to a variable that will receive the number of bytes required +** for the NOP command. If 'Bytes' is gcvNULL, nothing will be returned. +*/ +gceSTATUS +gckWLFE_Nop( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctSIZE_T * Bytes + ) +{ + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x *Bytes=%lu", + Hardware, Logical, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + + gcmkASSERT(Hardware->wlFE); + + if (Logical != gcvNULL) + { + if (*Bytes < 8) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + /* Append NOP. */ + logical[0] = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x03 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, "0x%x: NOP", Logical); + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the NOP command. */ + *Bytes = 8; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckWLFE_Event +** +** Append an EVENT command at the specified location in the command queue. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to an gckHARDWARE object. +** +** gctPOINTER Logical +** Pointer to the current location inside the command queue to append +** the EVENT command at or gcvNULL just to query the size of the EVENT +** command. +** +** gctUINT8 Event +** Event ID to program. +** +** gceKERNEL_WHERE FromWhere +** Location of the pipe to send the event. +** +** gctSIZE_T * Bytes +** Pointer to the number of bytes available for the EVENT command. If +** 'Logical' is gcvNULL, this argument will be ignored. +** +** OUTPUT: +** +** gctSIZE_T * Bytes +** Pointer to a variable that will receive the number of bytes required +** for the EVENT command. If 'Bytes' is gcvNULL, nothing will be +** returned. +*/ +gceSTATUS +gckWLFE_Event( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT8 Event, + IN gceKERNEL_WHERE FromWhere, + IN OUT gctUINT32 * Bytes + ) +{ + gctUINT size; + gctUINT32 destination = 0; + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + gctBOOL blt; + gctBOOL extraEventStates; + gctBOOL multiCluster; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x Event=%u FromWhere=%d *Bytes=%lu", + Hardware, Logical, Event, FromWhere, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + gcmkVERIFY_ARGUMENT(Event < 32); + + gcmkASSERT(Hardware->wlFE); + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_BLT_ENGINE)) + { + /* Send all event from blt. */ + if (FromWhere == gcvKERNEL_PIXEL) + { + FromWhere = gcvKERNEL_BLT; + } + } + + blt = FromWhere == gcvKERNEL_BLT ? gcvTRUE : gcvFALSE; + + multiCluster = gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_MULTI_CLUSTER); + + /* Determine the size of the command. */ + + extraEventStates = Hardware->extraEventStates && (FromWhere == gcvKERNEL_PIXEL); + + size = extraEventStates + ? gcmALIGN(8 + (1 + 5) * 4, 8) /* EVENT + 5 STATES */ + : 8; + + if (blt) + { + size += 16; + if (multiCluster) + size += 8; + } + + if (Logical != gcvNULL) + { + if (*Bytes < size) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + switch (FromWhere) + { + case gcvKERNEL_COMMAND: + /* From command processor. */ + destination = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 5:5) - (0 ? + 5:5) + 1))))))) << (0 ? + 5:5))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 5:5) - (0 ? + 5:5) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 5:5) - (0 ? 5:5) + 1))))))) << (0 ? 5:5))); + break; + + case gcvKERNEL_PIXEL: + /* From pixel engine. */ + destination = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 6:6) - (0 ? + 6:6) + 1))))))) << (0 ? + 6:6))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 6:6) - (0 ? + 6:6) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 6:6) - (0 ? 6:6) + 1))))))) << (0 ? 6:6))); + break; + + case gcvKERNEL_BLT: + destination = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:7) - (0 ? + 7:7) + 1))))))) << (0 ? + 7:7))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 7:7) - (0 ? + 7:7) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:7) - (0 ? 7:7) + 1))))))) << (0 ? 7:7))); + break; + + default: + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + if (blt) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + + if (multiCluster) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x50CE) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 7:0) - (0 ? + 7:0) + 1))))))) << (0 ? + 7:0))) | (((gctUINT32) ((gctUINT32) (Hardware->identity.clusterAvailMask & Hardware->options.userClusterMask) & ((gctUINT32) ((((1 ? + 7:0) - (0 ? + 7:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 7:0) - (0 ? 7:0) + 1))))))) << (0 ? 7:0))); + } + } + + /* Append EVENT(Event, destination). */ + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0E01) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + + *logical++ + = ((((gctUINT32) (destination)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 4:0) - (0 ? + 4:0) + 1))))))) << (0 ? + 4:0))) | (((gctUINT32) ((gctUINT32) (Event) & ((gctUINT32) ((((1 ? + 4:0) - (0 ? + 4:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 4:0) - (0 ? 4:0) + 1))))))) << (0 ? 4:0))); + + if (blt) + { + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (1) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x502E) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + *logical++ + = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 0:0) - (0 ? + 0:0) + 1))))))) << (0 ? + 0:0))) | (((gctUINT32) (0x0 & ((gctUINT32) ((((1 ? + 0:0) - (0 ? + 0:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 0:0) - (0 ? 0:0) + 1))))))) << (0 ? 0:0))); + } + + + /* Make sure the event ID gets written out before GPU can access it. */ + gcmkONERROR( + gckOS_MemoryBarrier(Hardware->os, logical + 1)); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + { + gctPHYS_ADDR_T phys; + gckOS_GetPhysicalAddress(Hardware->os, Logical, &phys); + gckOS_CPUPhysicalToGPUPhysical(Hardware->os, phys, &phys); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "0x%08x: EVENT %d", phys, Event); + } +#endif + + /* Append the extra states. These are needed for the chips that do not + ** support back-to-back events due to the async interface. The extra + ** states add the necessary delay to ensure that event IDs do not + ** collide. */ + if (extraEventStates) + { + *logical++ = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x01 & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) (0x0100) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 25:16) - (0 ? + 25:16) + 1))))))) << (0 ? + 25:16))) | (((gctUINT32) ((gctUINT32) (5) & ((gctUINT32) ((((1 ? + 25:16) - (0 ? + 25:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1))))))) << (0 ? 25:16))); + *logical++ = 0; + *logical++ = 0; + *logical++ = 0; + *logical++ = 0; + *logical++ = 0; + } + +#if gcdINTERRUPT_STATISTIC + if (Event < gcmCOUNTOF(Hardware->kernel->eventObj->queues)) + { + gckOS_AtomSetMask(Hardware->pendingEvent, 1 << Event); + } +#endif + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the EVENT command. */ + *Bytes = size; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckWLFE_ChipEnable( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gceCORE_3D_MASK ChipEnable, + IN OUT gctSIZE_T * Bytes + ) +{ + gckOS os = Hardware->os; + gctUINT32_PTR logical = (gctUINT32_PTR) Logical; + gceSTATUS status; + + gcmkHEADER_ARG("Hardware=0x%x Logical=0x%x ChipEnable=0x%x *Bytes=%lu", + Hardware, Logical, ChipEnable, gcmOPT_VALUE(Bytes)); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT((Logical == gcvNULL) || (Bytes != gcvNULL)); + + gcmkASSERT(Hardware->wlFE); + + if (Logical != gcvNULL) + { + if (*Bytes < 8) + { + /* Command queue too small. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + + /* Append CHIPENABLE. */ + gcmkWRITE_MEMORY( + logical, + ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? + 31:27) + 1))))))) << (0 ? + 31:27))) | (((gctUINT32) (0x0D & ((gctUINT32) ((((1 ? + 31:27) - (0 ? + 31:27) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 31:27) - (0 ? 31:27) + 1))))))) << (0 ? 31:27))) | ChipEnable + ); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, "0x%x: CHIPENABLE 0x%x", Logical, ChipEnable); + } + + if (Bytes != gcvNULL) + { + /* Return number of bytes required by the CHIPENABLE command. */ + *Bytes = 8; + } + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckWLFE_Execute +** +** Kickstart the hardware's command processor with an initialized command +** buffer. +** +** INPUT: +** +** gckHARDWARE Hardware +** Pointer to the gckHARDWARE object. +** +** gctUINT32 Address +** Hardware address of command buffer. +** +** gctUINT32 Bytes +** Number of bytes for the prefetch unit (until after the first LINK). +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckWLFE_Execute( + IN gckHARDWARE Hardware, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ) +{ + gceSTATUS status; + gctUINT32 control; + + gcmkHEADER_ARG("Hardware=0x%x Address=0x%x Bytes=%lu", + Hardware, Address, Bytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + gcmkASSERT(Hardware->wlFE); + + /* Enable all events. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00014, ~0U)); + + /* Write address register. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00654, Address)); + + /* Build control register. */ + control = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) ((Bytes + 7) >> 3) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + /* Set big endian */ + if (Hardware->bigEndian) + { + control |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 21:20) - (0 ? + 21:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 21:20) - (0 ? + 21:20) + 1))))))) << (0 ? + 21:20))) | (((gctUINT32) (0x2 & ((gctUINT32) ((((1 ? + 21:20) - (0 ? + 21:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 21:20) - (0 ? 21:20) + 1))))))) << (0 ? 21:20))); + } + + /* Make sure writing to command buffer and previous AHB register is done. */ + gcmkONERROR(gckOS_MemoryBarrier(Hardware->os, gcvNULL)); + + /* Write control register. */ + switch (Hardware->options.secureMode) + { + case gcvSECURE_NONE: + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00658, control)); + break; + case gcvSECURE_IN_NORMAL: + +#if defined(__KERNEL__) + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00658, control)); +#endif + gcmkONERROR( + gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x003A4, control)); + break; +#if gcdENABLE_TRUST_APPLICATION + case gcvSECURE_IN_TA: + /* Send message to TA. */ + gcmkONERROR(gckKERNEL_SecurityStartCommand(Hardware->kernel, Address, (gctUINT32)Bytes)); + break; +#endif + default: + break; + } + + /* Increase execute count. */ + Hardware->executeCount++; + + /* Record last execute address. */ + Hardware->lastExecuteAddress = Address; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HARDWARE, + "Started command buffer @ 0x%08x", + Address); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckWLFE_AtomicExecute( + IN gckHARDWARE Hardware, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ) +{ + gctUINT32 control; + gceSTATUS status = gcvSTATUS_OK; + + /* Enable all events. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00014, ~0U)); + + /* Write address register. */ + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00654, Address)); + + /* Build control register. */ + control = ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 16:16) - (0 ? + 16:16) + 1))))))) << (0 ? + 16:16))) | (((gctUINT32) (0x1 & ((gctUINT32) ((((1 ? + 16:16) - (0 ? + 16:16) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 16:16) - (0 ? 16:16) + 1))))))) << (0 ? 16:16))) + | ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 15:0) - (0 ? + 15:0) + 1))))))) << (0 ? + 15:0))) | (((gctUINT32) ((gctUINT32) ((Bytes + 7) >> 3) & ((gctUINT32) ((((1 ? + 15:0) - (0 ? + 15:0) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1))))))) << (0 ? 15:0))); + + /* Set big endian */ + if (Hardware->bigEndian) + { + control |= ((((gctUINT32) (0)) & ~(((gctUINT32) (((gctUINT32) ((((1 ? + 21:20) - (0 ? + 21:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? + 21:20) - (0 ? + 21:20) + 1))))))) << (0 ? + 21:20))) | (((gctUINT32) (0x2 & ((gctUINT32) ((((1 ? + 21:20) - (0 ? + 21:20) + 1) == 32) ? + ~0U : (~(~0U << ((1 ? 21:20) - (0 ? 21:20) + 1))))))) << (0 ? 21:20))); + } + + /* Make sure writing to command buffer and previous AHB register is done. */ + gcmkONERROR(gckOS_MemoryBarrier(Hardware->os, gcvNULL)); + + /* Write control register. */ + switch (Hardware->options.secureMode) + { + case gcvSECURE_NONE: + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00658, control)); + break; + case gcvSECURE_IN_NORMAL: + +#if defined(__KERNEL__) + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x00658, control)); +#endif + gcmkONERROR(gckOS_WriteRegisterEx(Hardware->os, Hardware->core, 0x003A4, control)); + break; +#if gcdENABLE_TRUST_APPLICATION + case gcvSECURE_IN_TA: + /* Send message to TA. */ + gcmkONERROR(gckKERNEL_SecurityStartCommand(Hardware->kernel, Address, (gctUINT32)Bytes)); + break; +#endif + default: + break; + } + + /* Increase execute count. */ + Hardware->executeCount++; + + /* Record last execute address. */ + Hardware->lastExecuteAddress = Address; + + /* Success. */ + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + return status; +} + +
diff --git a/hal/kernel/arch/gc_hal_kernel_recorder.c b/hal/kernel/arch/gc_hal_kernel_recorder.c new file mode 100644 index 0000000..0bdad09 --- /dev/null +++ b/hal/kernel/arch/gc_hal_kernel_recorder.c
@@ -0,0 +1,721 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal.h" +#include "gc_hal_kernel.h" +#include "gc_hal_kernel_context.h" + +/* + * ----------------------- + * HARDWARE STATE RECORDER + * ----------------------- + * + * State mirror buffer is used to 'mirror' hardware states since hardware + * states can't be dumpped. It is a context buffer which stores 'global' + * context. + * + * For each commit, state recorder + * 1) Records context buffer (if there is) and command buffers in this commit. + * 2) Parse those buffers to estimate the state changed. + * 3) Stores result to a mirror buffer. + * + * == Commit 0 ==================================================================== + * + * Context Buffer 0 + * + * Command Buffer 0 + * + * Mirror Buffer 0 <- Context Buffer 0 + Command Buffer 0 + * + * == Commit 1 ==================================================================== + * + * Command Buffer 1 + * + * Mirror Buffer 1 <- Command buffer 1 + Mirror Buffer 0 + * + * == Commit 2 ==================================================================== + * + * Context Buffer 2 (optional) + * + * Command Buffer 2 + * + * Mirror Buffer 2 <- Command buffer 2 + Context Buffer 2 + Mirror Buffer 1 + * + * == Commit N ==================================================================== + * + * For Commit N, these buffers are needed to reproduce hardware's behavior in + * this commit. + * + * Mirror Buffer [N - 1] : State Mirror accumlated by past commits, + * which is used to restore hardware state. + * Context Buffer [N] : + * Command Buffer [N] : Command buffer executed by hardware in this commit. + * + * If sequence of states programming matters, hardware's behavior can't be reproduced, + * but the state values stored in mirror buffer are assuring. + */ + +/* Queue size. */ +#define gcdNUM_RECORDS 6 + +typedef struct _gcsPARSER_HANDLER * gckPARSER_HANDLER; + +typedef void +(*HandlerFunction)( + IN gckPARSER_HANDLER Handler, + IN gctUINT32 Addr, + IN gctUINT32 Data + ); + +typedef struct _gcsPARSER_HANDLER +{ + gctUINT32 type; + gctUINT32 cmd; + gctPOINTER private; + HandlerFunction function; +} +gcsPARSER_HANDLER; + +typedef struct _gcsPARSER * gckPARSER; +typedef struct _gcsPARSER +{ + gctUINT8_PTR currentCmdBufferAddr; + + /* Current command. */ + gctUINT32 lo; + gctUINT32 hi; + + gctUINT8 cmdOpcode; + gctUINT16 cmdAddr; + gctUINT32 cmdSize; + gctUINT32 cmdRectCount; + gctUINT8 skip; + gctUINT32 skipCount; + + gctBOOL allow; + gctBOOL stop; + + /* Callback used by parser to handle a command. */ + gckPARSER_HANDLER commandHandler; +} +gcsPARSER; + +typedef struct _gcsMIRROR +{ + gctUINT32_PTR logical[gcdNUM_RECORDS]; + gctUINT32 bytes; + gcsSTATE_MAP_PTR map; + gctSIZE_T maxState; +} +gcsMIRROR; + +typedef struct _gcsDELTA +{ + gctUINT64 commitStamp; + gctUINT32_PTR command; + gctUINT32 commandBytes; + gctUINT32_PTR context; + gctUINT32 contextBytes; +} +gcsDELTA; + +typedef struct _gcsRECORDER +{ + gckOS os; + gcsMIRROR mirror; + gcsDELTA deltas[gcdNUM_RECORDS]; + + /* Index of current record. */ + gctUINT index; + + /* Number of records. */ + gctUINT num; + + /* Plugin used by gckPARSER. */ + gcsPARSER_HANDLER recorderHandler; + gckPARSER parser; +} +gcsRECORDER; + + +/******************************************************************************\ +***************************** Command Buffer Parser **************************** +\******************************************************************************/ + +/* +** Command buffer parser checks command buffer in FE's view to make sure there +** is no format error. +** +** Parser provide a callback mechnisam, so plug-in can be added to implement +** other functions. +*/ + +static void +_HandleLoadState( + IN OUT gckPARSER Parser + ) +{ + gctUINT i; + gctUINT32_PTR data = (gctUINT32_PTR)Parser->currentCmdBufferAddr; + gctUINT32 cmdAddr = Parser->cmdAddr; + + if (Parser->commandHandler == gcvNULL + || Parser->commandHandler->cmd != 0x01 + ) + { + /* No handler for this command. */ + return; + } + + for (i = 0; i < Parser->cmdSize; i++) + { + Parser->commandHandler->function(Parser->commandHandler, cmdAddr, *data); + + /* Advance to next state. */ + cmdAddr++; + data++; + } +} + +static void +_GetCommand( + IN OUT gckPARSER Parser + ) +{ + gctUINT32 * buffer = (gctUINT32 *)Parser->currentCmdBufferAddr; + + gctUINT16 cmdRectCount; + gctUINT16 cmdDataCount; + + Parser->hi = buffer[0]; + Parser->lo = buffer[1]; + + Parser->cmdOpcode = (((((gctUINT32) (Parser->hi)) >> (0 ? 31:27)) & ((gctUINT32) ((((1 ? 31:27) - (0 ? 31:27) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 31:27) - (0 ? 31:27) + 1)))))) ); + Parser->cmdRectCount = 1; + + switch (Parser->cmdOpcode) + { + case 0x01: + /* Extract count. */ + Parser->cmdSize = (((((gctUINT32) (Parser->hi)) >> (0 ? 25:16)) & ((gctUINT32) ((((1 ? 25:16) - (0 ? 25:16) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 25:16) - (0 ? 25:16) + 1)))))) ); + if (Parser->cmdSize == 0) + { + /* 0 means 1024. */ + Parser->cmdSize = 1024; + } + Parser->skip = (Parser->cmdSize & 0x1) ? 0 : 1; + + /* Extract address. */ + Parser->cmdAddr = (((((gctUINT32) (Parser->hi)) >> (0 ? 15:0)) & ((gctUINT32) ((((1 ? 15:0) - (0 ? 15:0) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 15:0) - (0 ? 15:0) + 1)))))) ); + + Parser->currentCmdBufferAddr = Parser->currentCmdBufferAddr + 4; + Parser->skipCount = Parser->cmdSize + Parser->skip; + break; + + case 0x05: + Parser->cmdSize = 4; + Parser->skipCount = gcmALIGN(Parser->cmdSize, 2); + break; + + case 0x06: + Parser->cmdSize = 5; + Parser->skipCount = gcmALIGN(Parser->cmdSize, 2); + break; + + case 0x0C: + Parser->cmdSize = 3; + Parser->skipCount = gcmALIGN(Parser->cmdSize, 2); + break; + + case 0x09: + Parser->cmdSize = 2; + Parser->cmdAddr = 0x0F16; + Parser->skipCount = gcmALIGN(Parser->cmdSize, 2); + break; + + case 0x04: + Parser->cmdSize = 1; + Parser->cmdAddr = 0x0F06; + + cmdRectCount = (((((gctUINT32) (Parser->hi)) >> (0 ? 15:8)) & ((gctUINT32) ((((1 ? 15:8) - (0 ? 15:8) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 15:8) - (0 ? 15:8) + 1)))))) ); + cmdDataCount = (((((gctUINT32) (Parser->hi)) >> (0 ? 26:16)) & ((gctUINT32) ((((1 ? 26:16) - (0 ? 26:16) + 1) == 32) ? ~0U : (~(~0U << ((1 ? 26:16) - (0 ? 26:16) + 1)))))) ); + + Parser->skipCount = gcmALIGN(Parser->cmdSize, 2) + + cmdRectCount * 2 + + gcmALIGN(cmdDataCount, 2); + + Parser->cmdRectCount = cmdRectCount; + break; + + case 0x03: + Parser->currentCmdBufferAddr = Parser->currentCmdBufferAddr + 8; + Parser->skipCount = 0; + break; + + case 0x02: + Parser->currentCmdBufferAddr = Parser->currentCmdBufferAddr + 8; + Parser->skipCount = 0; + break; + + case 0x07: + Parser->currentCmdBufferAddr = Parser->currentCmdBufferAddr + 8; + Parser->skipCount = 0; + break; + + case 0x08: + /* Commands after LINK isn't executed, skip them. */ + Parser->stop = gcvTRUE; + break; + + default: + /* Unknown command is a risk. */ + Parser->allow = gcvFALSE; + break; + } +} + +static void +_ParseCommand( + IN OUT gckPARSER Parser + ) +{ + switch(Parser->cmdOpcode) + { + case 0x01: + _HandleLoadState(Parser); + break; + case 0x05: + case 0x06: + case 0x0C: + break; + case 0x04: + break; + default: + break; + } + + /* Advance to next command. */ + Parser->currentCmdBufferAddr = Parser->currentCmdBufferAddr + + (Parser->skipCount << 2); +} + +gceSTATUS +gckPARSER_Parse( + IN gckPARSER Parser, + IN gctUINT8_PTR Buffer, + IN gctUINT32 Bytes + ) +{ + gckPARSER parser = Parser; + gctUINT8_PTR end = (gctUINT8_PTR)Buffer + Bytes; + + /* Initialize parser. */ + parser->currentCmdBufferAddr = (gctUINT8_PTR)Buffer; + parser->skip = 0; + parser->allow = gcvTRUE; + parser->stop = gcvFALSE; + + /* Go through command buffer until reaching the end + ** or meeting an error. */ + do + { + _GetCommand(parser); + + _ParseCommand(parser); + } + while ((parser->currentCmdBufferAddr < end) + && (parser->allow == gcvTRUE) + && (parser->stop == gcvFALSE) + ); + + if (parser->allow == gcvFALSE) + { + /* Error detected. */ + return gcvSTATUS_NOT_SUPPORTED; + } + + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckPARSER_RegisterCommandHandler +** +** Register a command handler which will be called when parser get a command. +** +*/ +gceSTATUS +gckPARSER_RegisterCommandHandler( + IN gckPARSER Parser, + IN gckPARSER_HANDLER Handler + ) +{ + Parser->commandHandler = Handler; + + return gcvSTATUS_OK; +} + +gceSTATUS +gckPARSER_Construct( + IN gckOS Os, + IN gckPARSER_HANDLER Handler, + OUT gckPARSER * Parser + ) +{ + gceSTATUS status; + gckPARSER pointer; + + gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcsPARSER), (gctPOINTER *)&pointer)); + + /* Put it here temp, should have a more general plug-in mechnisam. */ + pointer->commandHandler = Handler; + + *Parser = pointer; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +void +gckPARSER_Destroy( + IN gckOS Os, + IN gckPARSER Parser + ) +{ + gcmkOS_SAFE_FREE(Os, Parser); +} + +/******************************************************************************\ +**************************** Hardware States Recorder ************************** +\******************************************************************************/ + +static void +_RecodeState( + IN gckPARSER_HANDLER Handler, + IN gctUINT32 Addr, + IN gctUINT32 Data + ) +{ + gcmkVERIFY_OK(gckRECORDER_UpdateMirror(Handler->private, Addr, Data)); +} + +static gctUINT +_Previous( + IN gctUINT Index + ) +{ + if (Index == 0) + { + return gcdNUM_RECORDS - 1; + } + + return Index - 1; +} + +static gctUINT +_Next( + IN gctUINT Index + ) +{ + return (Index + 1) % gcdNUM_RECORDS; +} + +gceSTATUS +gckRECORDER_Construct( + IN gckOS Os, + IN gckHARDWARE Hardware, + OUT gckRECORDER * Recorder + ) +{ + gceSTATUS status; + gckCONTEXT context = gcvNULL; + gckRECORDER recorder = gcvNULL; + gctSIZE_T mapSize; + gctUINT i; + + gcmkONERROR(gckCONTEXT_Construct(Os, Hardware, 0, &context)); + + gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcsRECORDER), (gctPOINTER *)&recorder)); + + gckOS_ZeroMemory(recorder, gcmSIZEOF(gcsRECORDER)); + + /* Copy state map. */ + recorder->mirror.maxState = context->maxState; + + mapSize = context->maxState * gcmSIZEOF(gcsSTATE_MAP); + + gcmkONERROR(gckOS_Allocate(Os, mapSize, (gctPOINTER *)&recorder->mirror.map)); + + gckOS_MemCopy(recorder->mirror.map, context->map, mapSize); + + /* Copy context buffer. */ + recorder->mirror.bytes = context->totalSize; + + for (i = 0; i < gcdNUM_RECORDS; i++) + { + gcmkONERROR(gckOS_Allocate(Os, context->totalSize, (gctPOINTER *)&recorder->mirror.logical[i])); + gckOS_MemCopy(recorder->mirror.logical[i], context->buffer->logical, context->totalSize); + } + + for (i = 0; i < gcdNUM_RECORDS; i++) + { + gcmkONERROR(gckOS_Allocate(Os, gcdCMD_BUFFER_SIZE, (gctPOINTER *)&recorder->deltas[i].command)); + gcmkONERROR(gckOS_Allocate(Os, context->totalSize, (gctPOINTER *)&recorder->deltas[i].context)); + } + + recorder->index = 0; + recorder->num = 0; + + /* Initialize Parser plugin. */ + recorder->recorderHandler.cmd = 0x01; + recorder->recorderHandler.private = recorder; + recorder->recorderHandler.function = _RecodeState; + + gcmkONERROR(gckPARSER_Construct(Os, &recorder->recorderHandler, &recorder->parser)); + + recorder->os = Os; + + *Recorder = recorder; + + gckCONTEXT_Destroy(context); + return gcvSTATUS_OK; + +OnError: + if (context) + { + gckCONTEXT_Destroy(context); + } + + if (recorder) + { + gckRECORDER_Destory(Os, recorder); + } + + return status; +} + +gceSTATUS +gckRECORDER_Destory( + IN gckOS Os, + IN gckRECORDER Recorder + ) +{ + gctUINT i; + + if (Recorder->mirror.map) + { + gcmkOS_SAFE_FREE(Os, Recorder->mirror.map); + } + + for (i = 0; i < gcdNUM_RECORDS; i++) + { + if (Recorder->mirror.logical[i]) + { + gcmkOS_SAFE_FREE(Os, Recorder->mirror.logical[i]); + } + } + + for (i = 0; i < gcdNUM_RECORDS; i++) + { + if (Recorder->deltas[i].command) + { + gcmkOS_SAFE_FREE(Os, Recorder->deltas[i].command); + } + + if (Recorder->deltas[i].context) + { + gcmkOS_SAFE_FREE(Os, Recorder->deltas[i].context); + } + } + + if (Recorder->parser) + { + gckPARSER_Destroy(Os, Recorder->parser); + } + + gcmkOS_SAFE_FREE(Os, Recorder); + + return gcvSTATUS_OK; +} + +gceSTATUS +gckRECORDER_UpdateMirror( + IN gckRECORDER Recorder, + IN gctUINT32 State, + IN gctUINT32 Data + ) +{ + gctUINT32 index; + gcsSTATE_MAP_PTR map = Recorder->mirror.map; + gctUINT32_PTR buffer = Recorder->mirror.logical[Recorder->index]; + + if (State >= Recorder->mirror.maxState) + { + /* Ignore them just like HW does. */ + return gcvSTATUS_OK; + } + + index = map[State].index; + + if (index) + { + buffer[index] = Data; + } + + return gcvSTATUS_OK; +} + +void +gckRECORDER_AdvanceIndex( + IN gckRECORDER Recorder, + IN gctUINT64 CommitStamp + ) +{ + /* Get next record. */ + gctUINT next = (Recorder->index + 1) % gcdNUM_RECORDS; + + /* Record stamp of this commit. */ + Recorder->deltas[Recorder->index].commitStamp = CommitStamp; + + /* Mirror of next record is mirror of this record and delta in next record. */ + gckOS_MemCopy(Recorder->mirror.logical[next], + Recorder->mirror.logical[Recorder->index], Recorder->mirror.bytes); + + /* Advance to next record. */ + Recorder->index = next; + + Recorder->num = gcmMIN(Recorder->num + 1, gcdNUM_RECORDS - 1); + + + /* Reset delta. */ + Recorder->deltas[Recorder->index].commandBytes = 0; + Recorder->deltas[Recorder->index].contextBytes = 0; +} + +void +gckRECORDER_Record( + IN gckRECORDER Recorder, + IN gctUINT8_PTR CommandBuffer, + IN gctUINT32 CommandBytes, + IN gctUINT8_PTR ContextBuffer, + IN gctUINT32 ContextBytes + ) +{ + gcsDELTA * delta = &Recorder->deltas[Recorder->index]; + + if (CommandBytes != 0xFFFFFFFF) + { + gckPARSER_Parse(Recorder->parser, CommandBuffer, CommandBytes); + gckOS_MemCopy(delta->command, CommandBuffer, CommandBytes); + delta->commandBytes = CommandBytes; + } + + if (ContextBytes != 0xFFFFFFFF) + { + gckPARSER_Parse(Recorder->parser, ContextBuffer, ContextBytes); + gckOS_MemCopy(delta->context, ContextBuffer, ContextBytes); + delta->contextBytes = ContextBytes; + } +} + +void +gckRECORDER_Dump( + IN gckRECORDER Recorder + ) +{ + gctUINT last = Recorder->index; + gctUINT previous; + gctUINT i; + gcsMIRROR *mirror = &Recorder->mirror; + gcsDELTA *delta; + gckOS os = Recorder->os; + + for (i = 0; i < Recorder->num; i++) + { + last = _Previous(last); + } + + for (i = 0; i < Recorder->num; i++) + { + delta = &Recorder->deltas[last]; + + /* Dump record */ + gcmkPRINT("#[commit %llu]", delta->commitStamp); + + if (delta->commitStamp) + { + previous = _Previous(last); + + gcmkPRINT("#[mirror]"); + gckOS_DumpBuffer(os, gcvDUMP_BUFFER_KERNEL_CONTEXT, mirror->logical[previous], ~0U, mirror->bytes); + gcmkPRINT("#[kernel.execute]"); + } + + if (delta->contextBytes) + { + gckOS_DumpBuffer(os, gcvDUMP_BUFFER_KERNEL_CONTEXT, delta->context, ~0U, delta->contextBytes); + gcmkPRINT("#[kernel.execute]"); + } + + gckOS_DumpBuffer(os, gcvDUMP_BUFFER_KERNEL_COMMAND, delta->command, ~0U, delta->commandBytes); + gcmkPRINT("#[kernel.execute]"); + + last = _Next(last); + } +} + +
diff --git a/hal/kernel/arch/makefile.linux b/hal/kernel/arch/makefile.linux new file mode 100644 index 0000000..de550f3 --- /dev/null +++ b/hal/kernel/arch/makefile.linux
@@ -0,0 +1,92 @@ +############################################################################## +# +# The MIT License (MIT) +# +# Copyright (c) 2014 - 2020 Vivante Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +# +############################################################################## +# +# The GPL License (GPL) +# +# Copyright (C) 2014 - 2020 Vivante Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +############################################################################## +# +# Note: This software is released under dual MIT and GPL licenses. A +# recipient may use this file under the terms of either the MIT license or +# GPL License. If you wish to use only one license not the other, you can +# indicate your decision by deleting one of the above license notices in your +# version of this file. +# +############################################################################## + + +# +# Linux build file for architecture dependent kernel HAL layer. +# + +################################################################################ +# Include common definitions. + +include $(AQROOT)/makefile.linux.def + +################################################################################ +# Define a shortcut for the main target. + +STATIC = 1 +TARGET_NAME = libhalarchkernel.a + +################################################################################ +# Supply additional include directories. + +INCLUDE += -I$(AQROOT)/hal/kernel/inc +INCLUDE += -I$(AQROOT)/hal/kernel/inc +INCLUDE += -I$(AQROOT)/hal/kernel +INCLUDE += -I$(AQROOT)/hal/kernel/arch + +CFLAGS += $(INCLUDE) -Werror -ansi + + +################################################################################ +# Describe object files. + +OBJECTS = $(OBJ_DIR)/gc_hal_kernel_context.o \ + $(OBJ_DIR)/gc_hal_kernel_hardware.o \ + $(OBJ_DIR)/gc_hal_kernel_hardware_func.o \ + $(OBJ_DIR)/gc_hal_kernel_hardware_async_fe.o \ + $(OBJ_DIR)/gc_hal_kernel_hardware_mc_fe.o \ + $(OBJ_DIR)/gc_hal_kernel_hardware_waitlink_fe.o + +include $(AQROOT)/common.target
diff --git a/hal/kernel/gc_hal_kernel.c b/hal/kernel/gc_hal_kernel.c new file mode 100644 index 0000000..93634cf --- /dev/null +++ b/hal/kernel/gc_hal_kernel.c
@@ -0,0 +1,5759 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" + +#if gcdDEC_ENABLE_AHB +#include "viv_dec300_main.h" +#endif + +#if gcdCAPTURE_ONLY_MODE +#include "arch/gc_hal_kernel_context.h" +#endif + +#define _GC_OBJ_ZONE gcvZONE_KERNEL + +/******************************************************************************* +***** Version Signature *******************************************************/ + +#define _gcmTXT2STR(t) #t +#define gcmTXT2STR(t) _gcmTXT2STR(t) +const char * _VERSION = "\n\0$VERSION$" + gcmTXT2STR(gcvVERSION_MAJOR) "." + gcmTXT2STR(gcvVERSION_MINOR) "." + gcmTXT2STR(gcvVERSION_PATCH) ":" + gcmTXT2STR(gcvVERSION_BUILD) "$\n"; + +/******************************************************************************\ +******************************* gckKERNEL API Code ****************************** +\******************************************************************************/ + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) +#define gcmDEFINE2TEXT(d) #d +gctCONST_STRING _DispatchText[] = +{ + gcmDEFINE2TEXT(gcvHAL_CHIP_INFO), + gcmDEFINE2TEXT(gcvHAL_VERSION), + gcmDEFINE2TEXT(gcvHAL_SET_TIMEOUT), + gcmDEFINE2TEXT(gcvHAL_QUERY_VIDEO_MEMORY), + gcmDEFINE2TEXT(gcvHAL_QUERY_CHIP_IDENTITY), + gcmDEFINE2TEXT(gcvHAL_QUERY_CHIP_OPTION), + gcmDEFINE2TEXT(gcvHAL_QUERY_CHIP_FREQUENCY), + gcmDEFINE2TEXT(gcvHAL_ALLOCATE_NON_PAGED_MEMORY), + gcmDEFINE2TEXT(gcvHAL_FREE_NON_PAGED_MEMORY), + gcmDEFINE2TEXT(gcvHAL_ALLOCATE_LINEAR_VIDEO_MEMORY), + gcmDEFINE2TEXT(gcvHAL_WRAP_USER_MEMORY), + gcmDEFINE2TEXT(gcvHAL_RELEASE_VIDEO_MEMORY), + gcmDEFINE2TEXT(gcvHAL_LOCK_VIDEO_MEMORY), + gcmDEFINE2TEXT(gcvHAL_UNLOCK_VIDEO_MEMORY), + gcmDEFINE2TEXT(gcvHAL_BOTTOM_HALF_UNLOCK_VIDEO_MEMORY), + gcmDEFINE2TEXT(gcvHAL_EXPORT_VIDEO_MEMORY), + gcmDEFINE2TEXT(gcvHAL_NAME_VIDEO_MEMORY), + gcmDEFINE2TEXT(gcvHAL_IMPORT_VIDEO_MEMORY), + gcmDEFINE2TEXT(gcvHAL_MAP_MEMORY), + gcmDEFINE2TEXT(gcvHAL_UNMAP_MEMORY), + gcmDEFINE2TEXT(gcvHAL_CACHE), + gcmDEFINE2TEXT(gcvHAL_ATTACH), + gcmDEFINE2TEXT(gcvHAL_DETACH), + gcmDEFINE2TEXT(gcvHAL_EVENT_COMMIT), + gcmDEFINE2TEXT(gcvHAL_COMMIT), + gcmDEFINE2TEXT(gcvHAL_COMMIT_DONE), + gcmDEFINE2TEXT(gcvHAL_USER_SIGNAL), + gcmDEFINE2TEXT(gcvHAL_SIGNAL), + gcmDEFINE2TEXT(gcvHAL_WRITE_DATA), + gcmDEFINE2TEXT(gcvHAL_READ_REGISTER), + gcmDEFINE2TEXT(gcvHAL_WRITE_REGISTER), + gcmDEFINE2TEXT(gcvHAL_READ_REGISTER_EX), + gcmDEFINE2TEXT(gcvHAL_WRITE_REGISTER_EX), + gcmDEFINE2TEXT(gcvHAL_GET_PROFILE_SETTING), + gcmDEFINE2TEXT(gcvHAL_SET_PROFILE_SETTING), + gcmDEFINE2TEXT(gcvHAL_READ_PROFILER_REGISTER_SETTING), + gcmDEFINE2TEXT(gcvHAL_READ_ALL_PROFILE_REGISTERS_PART1), + gcmDEFINE2TEXT(gcvHAL_READ_ALL_PROFILE_REGISTERS_PART2), + gcmDEFINE2TEXT(gcvHAL_PROFILE_REGISTERS_2D), + gcmDEFINE2TEXT(gcvHAL_SET_POWER_MANAGEMENT_STATE), + gcmDEFINE2TEXT(gcvHAL_QUERY_POWER_MANAGEMENT_STATE), + gcmDEFINE2TEXT(gcvHAL_CONFIG_POWER_MANAGEMENT), + gcmDEFINE2TEXT(gcvHAL_GET_BASE_ADDRESS), + gcmDEFINE2TEXT(gcvHAL_SET_IDLE), + gcmDEFINE2TEXT(gcvHAL_RESET), + gcmDEFINE2TEXT(gcvHAL_SET_DEBUG_LEVEL_ZONE), + gcmDEFINE2TEXT(gcvHAL_DEBUG_DUMP), + gcmDEFINE2TEXT(gcvHAL_UPDATE_DEBUG_CALLBACK), + gcmDEFINE2TEXT(gcvHAL_CONFIG_CTX_FRAMEWORK), + gcmDEFINE2TEXT(gcvHAL_DUMP_GPU_STATE), + gcmDEFINE2TEXT(gcvHAL_DUMP_EVENT), + gcmDEFINE2TEXT(gcvHAL_DUMP_GPU_PROFILE), + gcmDEFINE2TEXT(gcvHAL_TIMESTAMP), + gcmDEFINE2TEXT(gcvHAL_DATABASE), + gcmDEFINE2TEXT(gcvHAL_GET_FRAME_INFO), + gcmDEFINE2TEXT(gcvHAL_QUERY_COMMAND_BUFFER), + gcmDEFINE2TEXT(gcvHAL_SET_FSCALE_VALUE), + gcmDEFINE2TEXT(gcvHAL_GET_FSCALE_VALUE), + gcmDEFINE2TEXT(gcvHAL_QUERY_RESET_TIME_STAMP), + gcmDEFINE2TEXT(gcvHAL_CREATE_NATIVE_FENCE), + gcmDEFINE2TEXT(gcvHAL_WAIT_NATIVE_FENCE), + gcmDEFINE2TEXT(gcvHAL_SHBUF), + gcmDEFINE2TEXT(gcvHAL_GET_GRAPHIC_BUFFER_FD), + gcmDEFINE2TEXT(gcvHAL_SET_VIDEO_MEMORY_METADATA), + gcmDEFINE2TEXT(gcvHAL_GET_VIDEO_MEMORY_FD), + gcmDEFINE2TEXT(gcvHAL_DESTROY_MMU), + gcmDEFINE2TEXT(gcvHAL_WAIT_FENCE), + gcmDEFINE2TEXT(gcvHAL_DEVICE_MUTEX), + gcmDEFINE2TEXT(gcvHAL_DEC200_TEST), + gcmDEFINE2TEXT(gcvHAL_DEC300_READ), + gcmDEFINE2TEXT(gcvHAL_DEC300_WRITE), + gcmDEFINE2TEXT(gcvHAL_DEC300_FLUSH), + gcmDEFINE2TEXT(gcvHAL_DEC300_FLUSH_WAIT), +}; +#endif + +#if gcdGPU_TIMEOUT && gcdINTERRUPT_STATISTIC +void +_MonitorTimerFunction( + gctPOINTER Data + ) +{ + gckKERNEL kernel = (gckKERNEL)Data; + gctINT32 pendingInterrupt; + gctBOOL reset = gcvFALSE; + gctINT32 mask; + gctUINT32 advance = kernel->timeOut/2; + + + if (kernel->monitorTimerStop) + { + /* Stop. */ + return; + } + + gckOS_AtomGet(kernel->os, kernel->eventObj->interruptCount, &pendingInterrupt); + + if (pendingInterrupt < 0) + { + gctINT i = 0 - pendingInterrupt; + gctINT pendingMask; + + gcmkVERIFY_OK(gckOS_AtomGet( + kernel->os, + kernel->hardware->pendingEvent, + &pendingMask + )); + + gcmkPRINT("[galcore]: Number of pending interrupt is %d mask is %x", + pendingInterrupt, pendingMask); + + while (i--) + { + /* Ignore counting which should not exist. */ + gckOS_AtomIncrement(kernel->os, kernel->eventObj->interruptCount, &pendingInterrupt); + } + + gckOS_AtomGet(kernel->os, kernel->eventObj->interruptCount, &pendingInterrupt); + } + + if (kernel->monitoring == gcvFALSE) + { + if (pendingInterrupt) + { + /* Begin to mointor GPU state. */ + kernel->monitoring = gcvTRUE; + + /* Record current state. */ + kernel->lastCommitStamp = kernel->eventObj->lastCommitStamp; + kernel->restoreAddress = kernel->hardware->lastWaitLink; + gcmkVERIFY_OK(gckOS_AtomGet( + kernel->os, + kernel->hardware->pendingEvent, + &kernel->restoreMask + )); + + /* Clear timeout. */ + kernel->timer = 0; + } + } + else + { + if (pendingInterrupt) + { + gcmkVERIFY_OK(gckOS_AtomGet( + kernel->os, + kernel->hardware->pendingEvent, + &mask + )); + + if (kernel->eventObj->lastCommitStamp == kernel->lastCommitStamp + && kernel->hardware->lastWaitLink == kernel->restoreAddress + && mask == kernel->restoreMask + ) + { + /* GPU state is not changed, accumlate timeout. */ + kernel->timer += advance; + + if (kernel->timer >= kernel->timeOut) + { + /* GPU stuck, trigger reset. */ + reset = gcvTRUE; + } + } + else + { + /* GPU state changed, cancel current timeout.*/ + kernel->monitoring = gcvFALSE; + } + } + else + { + /* GPU finish all jobs, cancel current timeout*/ + kernel->monitoring = gcvFALSE; + } + } + + if (reset) + { + gckKERNEL_Recovery(kernel); + + /* Work in this timeout is done. */ + kernel->monitoring = gcvFALSE; + } + + gcmkVERIFY_OK(gckOS_StartTimer(kernel->os, kernel->monitorTimer, advance)); +} +#endif + +void +_DumpDriverConfigure( + IN gckKERNEL Kernel + ) +{ + gcmkPRINT_N(0, "**************************\n"); + gcmkPRINT_N(0, "*** GPU DRV CONFIG ***\n"); + gcmkPRINT_N(0, "**************************\n"); + + gcmkPRINT("Galcore version %d.%d.%d.%d\n", + gcvVERSION_MAJOR, gcvVERSION_MINOR, gcvVERSION_PATCH, gcvVERSION_BUILD); + + gckOS_DumpParam(); +} + +void +_DumpState( + IN gckKERNEL Kernel + ) +{ + /* Dump GPU Debug registers. */ + gcmkVERIFY_OK(gckHARDWARE_DumpGPUState(Kernel->hardware)); + + /* Dump Pending event. */ + gcmkVERIFY_OK(gckEVENT_Dump(Kernel->eventObj)); + + /* Dump Process DB. */ + gcmkVERIFY_OK(gckKERNEL_DumpProcessDB(Kernel)); + +#if gcdRECORD_COMMAND + /* Dump record. */ + gckRECORDER_Dump(Kernel->command->recorder); +#endif + + if (Kernel->command) + { + gcmkVERIFY_OK(gckCOMMAND_DumpExecutingBuffer(Kernel->command)); + } +} + +gceSTATUS +gckKERNEL_GetHardwareType( + IN gckKERNEL Kernel, + OUT gceHARDWARE_TYPE *Type + ) +{ + gceHARDWARE_TYPE type; + gcmkHEADER(); + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + + { + type = Kernel->hardware->type; + } + + *Type = type; + + gcmkFOOTER_ARG("type=%d", type); + return gcvSTATUS_OK; +} + +gceSTATUS +_SetRecovery( + IN gckKERNEL Kernel, + IN gctBOOL Recovery, + IN gctUINT32 StuckDump + ) +{ + Kernel->recovery = Recovery; + + if (Recovery == gcvFALSE) + { + /* Dump stuck information if Recovery is disabled. */ + Kernel->stuckDump = gcmMAX(StuckDump, gcvSTUCK_DUMP_USER_COMMAND); + } + + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckKERNEL_Construct +** +** Construct a new gckKERNEL object. +** +** INPUT: +** +** gckOS Os +** Pointer to an gckOS object. +** +** gceCORE Core +** Specified core. +** +** IN gctPOINTER Context +** Pointer to a driver defined context. +** +** IN gckDB SharedDB, +** Pointer to a shared DB. +** +** OUTPUT: +** +** gckKERNEL * Kernel +** Pointer to a variable that will hold the pointer to the gckKERNEL +** object. +*/ + +gceSTATUS +gckKERNEL_Construct( + IN gckOS Os, + IN gceCORE Core, + IN gctUINT ChipID, + IN gctPOINTER Context, + IN gckDEVICE Device, + IN gckDB SharedDB, + OUT gckKERNEL * Kernel + ) +{ + gckKERNEL kernel = gcvNULL; + gceSTATUS status; + gctSIZE_T i; + gctPOINTER pointer = gcvNULL; + gctUINT64 data; + gctUINT32 recovery; + gctUINT32 stuckDump; + gctUINT64 dynamicMap = 1; + + gcmkHEADER_ARG("Os=%p Context=%p", Os, Context); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Os, gcvOBJ_OS); + gcmkVERIFY_ARGUMENT(Kernel != gcvNULL); + + /* Allocate the gckKERNEL object. */ + gcmkONERROR(gckOS_Allocate(Os, + gcmSIZEOF(struct _gckKERNEL), + &pointer)); + + /* Zero the object. */ + gckOS_ZeroMemory(pointer, gcmSIZEOF(struct _gckKERNEL)); + + kernel = pointer; + + /* Initialize the gckKERNEL object. */ + kernel->object.type = gcvOBJ_KERNEL; + kernel->os = Os; + kernel->core = Core; + kernel->device = Device; + kernel->chipID = ChipID; + kernel->threadInitialized = gcvTRUE; + +#if gcdENABLE_TRUST_APPLICATION + /* Connect to security service for this GPU. */ + gcmkONERROR(gckKERNEL_SecurityOpen(kernel, kernel->core, &kernel->securityChannel)); +#endif + + if (SharedDB == gcvNULL) + { + gcmkONERROR(gckOS_Allocate(Os, + gcmSIZEOF(struct _gckDB), + &pointer)); + + kernel->db = pointer; + kernel->dbCreated = gcvTRUE; + kernel->db->freeDatabase = gcvNULL; + kernel->db->freeRecord = gcvNULL; + kernel->db->dbMutex = gcvNULL; + kernel->db->lastDatabase = gcvNULL; + kernel->db->idleTime = 0; + kernel->db->lastIdle = 0; + kernel->db->lastSlowdown = 0; + + for (i = 0; i < gcmCOUNTOF(kernel->db->db); ++i) + { + kernel->db->db[i] = gcvNULL; + } + + /* Construct a database mutex. */ + gcmkONERROR(gckOS_CreateMutex(Os, &kernel->db->dbMutex)); + + /* Construct a video memory name database. */ + gcmkONERROR(gckKERNEL_CreateIntegerDatabase( + kernel, + 512, + &kernel->db->nameDatabase + )); + + /* Construct a video memory name database mutex. */ + gcmkONERROR(gckOS_CreateMutex(Os, &kernel->db->nameDatabaseMutex)); + + /* Construct a pointer name database. */ + gcmkONERROR(gckKERNEL_CreateIntegerDatabase( + kernel, + 512, + &kernel->db->pointerDatabase + )); + + /* Initialize video memory node list. */ + gcsLIST_Init(&kernel->db->videoMemList); + + gcmkONERROR(gckOS_CreateMutex(Os, &kernel->db->videoMemListMutex)); + } + else + { + kernel->db = SharedDB; + kernel->dbCreated = gcvFALSE; + } + + for (i = 0; i < gcmCOUNTOF(kernel->timers); ++i) + { + kernel->timers[i].startTime = 0; + kernel->timers[i].stopTime = 0; + } + + gcmkONERROR(gckOS_CreateMutex(Os, &kernel->vidMemBlockMutex)); + + /* Save context. */ + kernel->context = Context; + + /* Construct atom holding number of clients. */ + kernel->atomClients = gcvNULL; + gcmkONERROR(gckOS_AtomConstruct(Os, &kernel->atomClients)); + + kernel->recovery = gcvTRUE; + kernel->stuckDump = gcvSTUCK_DUMP_NONE; + + /* Override default recovery and stuckDump setting. */ + status = gckOS_QueryOption(Os, "recovery", &data); + recovery = (gctUINT32)data; + + if (gcmIS_SUCCESS(status)) + { + status = gckOS_QueryOption(Os, "stuckDump", &data); + stuckDump = (gctUINT32)data; + + gcmkASSERT(status == gcvSTATUS_OK); + + _SetRecovery(kernel, recovery, stuckDump); + } + + status = gckOS_QueryOption(Os, "sRAMLoopMode", &data); + kernel->sRAMLoopMode = (status == gcvSTATUS_OK) ? data : 0; + + /* Need the kernel reference before gckKERNEL_Construct() completes. + gckOS_MapPagesEx() is called to map kernel virtual command buffers. */ + *Kernel = kernel; + + { + /* Construct the gckHARDWARE object. */ + gcmkONERROR( + gckHARDWARE_Construct(Os, kernel->device, kernel->core, &kernel->hardware)); + + /* Set pointer to gckKERNEL object in gckHARDWARE object. */ + kernel->hardware->kernel = kernel; + + kernel->sRAMIndex = 0; + kernel->extSRAMIndex = 0; + + for (i = gcvSRAM_INTERNAL0; i < gcvSRAM_INTER_COUNT; i++) + { + kernel->sRAMVidMem[i] = kernel->hardware->sRAMVidMem[i]; + kernel->sRAMPhysical[i] = kernel->hardware->sRAMPhysical[i]; + kernel->sRAMPhysFaked[i] = gcvFALSE; + } + + kernel->timeOut = kernel->hardware->type == gcvHARDWARE_2D + ? gcdGPU_2D_TIMEOUT + : gcdGPU_TIMEOUT + ; + +#if gcdSHARED_PAGETABLE + /* Construct the gckMMU object. */ + gcmkONERROR( + gckMMU_Construct(kernel, gcdMMU_SIZE, &kernel->mmu)); +#else + if (Device == gcvNULL) + { + /* Construct the gckMMU object. */ + gcmkONERROR( + gckMMU_Construct(kernel, gcdMMU_SIZE, &kernel->mmu)); + } + else + { + gcmkONERROR(gckDEVICE_GetMMU(Device, kernel->hardware->type, &kernel->mmu)); + + if (kernel->mmu == gcvNULL) + { + gcmkONERROR( + gckMMU_Construct(kernel, gcdMMU_SIZE, &kernel->mmu)); + + gcmkONERROR( + gckDEVICE_SetMMU(Device, kernel->hardware->type, kernel->mmu)); + } + } + + gcmkONERROR( + gckMMU_SetupSRAM(kernel->mmu, kernel->hardware, kernel->device)); + + status = gckOS_QueryOption(Os, "mmuDynamicMap", &dynamicMap); + if (dynamicMap && kernel->hardware->mmuVersion && !kernel->mmu->dynamicAreaSetuped) + { + gcmkONERROR( + gckMMU_SetupDynamicSpace(kernel->mmu)); + + kernel->mmu->dynamicAreaSetuped = gcvTRUE; + } + + if (kernel->hardware->mmuVersion > 0) + { + /* Flush MTLB table. */ + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + kernel, + kernel->mmu->mtlbVideoMem, + 0, + kernel->mmu->mtlbLogical, + kernel->mmu->mtlbSize + )); + } +#endif + + kernel->contiguousBaseAddress = kernel->mmu->contiguousBaseAddress; + kernel->externalBaseAddress = kernel->mmu->externalBaseAddress; + + /* Construct the gckCOMMAND object, either MCFE or wait-link FE can exist. */ + if (gckHARDWARE_IsFeatureAvailable(kernel->hardware, gcvFEATURE_MCFE)) + { + /* Construct the gckCOMMAND object for multi-channel FE. */ + gcmkONERROR(gckCOMMAND_Construct(kernel, gcvHW_FE_MULTI_CHANNEL, &kernel->command)); + + /* Construct gckEVENT for multi-channel FE. */ + gcmkONERROR(gckEVENT_Construct(kernel, kernel->command, &kernel->eventObj)); + } + else + { + /* Construct the gckCOMMAND object for legacy wait-link FE. */ + gcmkONERROR(gckCOMMAND_Construct(kernel, gcvHW_FE_WAIT_LINK, &kernel->command)); + + /* Construct the gckEVENT object. */ + gcmkONERROR(gckEVENT_Construct(kernel, kernel->command, &kernel->eventObj)); + } + + if (gckHARDWARE_IsFeatureAvailable(kernel->hardware, gcvFEATURE_ASYNC_BLIT)) + { + /* Construct the gckCOMMAND object for BLT engine. */ + gcmkONERROR(gckCOMMAND_Construct(kernel, gcvHW_FE_ASYNC, &kernel->asyncCommand)); + + /* Construct gckEVENT for BLT. */ + gcmkONERROR(gckEVENT_Construct(kernel, kernel->asyncCommand, &kernel->asyncEvent)); + } + + gcmkVERIFY_OK(gckOS_GetTime(&kernel->resetTimeStamp)); + + /* Post construct hardware elements after MMU settle. */ + gcmkONERROR(gckHARDWARE_PostConstruct(kernel->hardware)); + + /* Initialize the GPU. */ + gcmkONERROR( + gckHARDWARE_InitializeHardware(kernel->hardware)); + +#if gcdDVFS + if (gckHARDWARE_IsFeatureAvailable(kernel->hardware, + gcvFEATURE_DYNAMIC_FREQUENCY_SCALING)) + { + gcmkONERROR(gckDVFS_Construct(kernel->hardware, &kernel->dvfs)); + gcmkONERROR(gckDVFS_Start(kernel->dvfs)); + } +#endif + +#if COMMAND_PROCESSOR_VERSION == 1 + if (kernel->command) + { + /* Start the command queue. */ + gcmkONERROR(gckCOMMAND_Start(kernel->command)); + } + + if (kernel->asyncCommand) + { + /* Start the async command queue. */ + gcmkONERROR(gckCOMMAND_Start(kernel->asyncCommand)); + } +#endif + } + +#if VIVANTE_PROFILER + /* Initialize profile setting */ + kernel->profileEnable = gcvFALSE; + kernel->profileCleanRegister = gcvTRUE; +#endif + +#if gcdLINUX_SYNC_FILE + gcmkONERROR(gckOS_CreateSyncTimeline(Os, Core, &kernel->timeline)); +#endif + +#if gcdSECURITY + /* Connect to security service for this GPU. */ + gcmkONERROR(gckKERNEL_SecurityOpen(kernel, kernel->core, &kernel->securityChannel)); +#endif + +#if gcdGPU_TIMEOUT && gcdINTERRUPT_STATISTIC + if (kernel->timeOut) + { + gcmkVERIFY_OK(gckOS_CreateTimer( + Os, + (gctTIMERFUNCTION)_MonitorTimerFunction, + (gctPOINTER)kernel, + &kernel->monitorTimer + )); + + kernel->monitoring = gcvFALSE; + + kernel->monitorTimerStop = gcvFALSE; + + gcmkVERIFY_OK(gckOS_StartTimer( + Os, + kernel->monitorTimer, + 100 + )); + } +#endif + + /* Return pointer to the gckKERNEL object. */ + *Kernel = kernel; + + /* Success. */ + gcmkFOOTER_ARG("*Kernel=%p", *Kernel); + return gcvSTATUS_OK; + +OnError: + gckOS_SetGPUPower(Os, kernel->core, gcvFALSE, gcvFALSE); + *Kernel = gcvNULL; + + if (kernel != gcvNULL) + { + gckKERNEL_Destroy(kernel); + } + + /* Return the error. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckKERNEL_Destroy +** +** Destroy an gckKERNEL object. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object to destroy. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_Destroy( + IN gckKERNEL Kernel + ) +{ + gctSIZE_T i; + gcsDATABASE_PTR database, databaseNext; + gcsDATABASE_RECORD_PTR record, recordNext; + + gcmkHEADER_ARG("Kernel=%p", Kernel); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); +#if QNX_SINGLE_THREADED_DEBUGGING + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->debugMutex)); +#endif + + if (Kernel->monitorTimer) + { + /* Stop and destroy monitor timer. */ + gcmkVERIFY_OK(gckOS_StopTimer(Kernel->os, Kernel->monitorTimer)); + gcmkVERIFY_OK(gckOS_DestroyTimer(Kernel->os, Kernel->monitorTimer)); + } + + { + if (Kernel->command) + { + /* Destroy the gckCOMMNAND object. */ + gcmkVERIFY_OK(gckCOMMAND_Destroy(Kernel->command)); + } + + if (Kernel->asyncCommand) + { + gcmkVERIFY_OK(gckCOMMAND_Destroy(Kernel->asyncCommand)); + } + + if (Kernel->asyncEvent) + { + gcmkVERIFY_OK(gckEVENT_Destroy(Kernel->asyncEvent)); + } + + if (Kernel->eventObj) + { + /* Destroy the gckEVENT object. */ + gcmkVERIFY_OK(gckEVENT_Destroy(Kernel->eventObj)); + } + + /* Destroy hardware resources before destroying MMU. */ + gcmkVERIFY_OK(gckHARDWARE_PreDestroy(Kernel->hardware)); + + if (Kernel->mmu) + { +#if gcdSHARED_PAGETABLE + /* Destroy the gckMMU object. */ + gcmkVERIFY_OK(gckMMU_Destroy(Kernel->mmu)); +#else + if (Kernel->mmu->hardware == Kernel->hardware) + { + /* Destroy the gckMMU object. */ + gcmkVERIFY_OK(gckMMU_Destroy(Kernel->mmu)); + } +#endif + } + + /* Destroy the gckHARDWARE object. */ + gcmkVERIFY_OK(gckHARDWARE_Destroy(Kernel->hardware)); + } + + if (Kernel->atomClients) + { + /* Detsroy the client atom. */ + gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, Kernel->atomClients)); + } + + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->vidMemBlockMutex)); + + /* Destroy the database. */ + if (Kernel->dbCreated) + { + for (i = 0; i < gcmCOUNTOF(Kernel->db->db); ++i) + { + if (Kernel->db->db[i] != gcvNULL) + { + gcmkVERIFY_OK( + gckKERNEL_DestroyProcessDB(Kernel, Kernel->db->db[i]->processID)); + } + } + + /* Free all databases. */ + for (database = Kernel->db->freeDatabase; + database != gcvNULL; + database = databaseNext) + { + databaseNext = database->next; + + if (database->counterMutex) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, database->counterMutex)); + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, database)); + } + + if (Kernel->db->lastDatabase != gcvNULL) + { + if (Kernel->db->lastDatabase->counterMutex) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->lastDatabase->counterMutex)); + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Kernel->db->lastDatabase)); + } + + /* Free all database records. */ + for (record = Kernel->db->freeRecord; record != gcvNULL; record = recordNext) + { + recordNext = record->next; + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, record)); + } + + if (Kernel->db->dbMutex) + { + /* Destroy the database mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->dbMutex)); + } + + if (Kernel->db->nameDatabase) + { + /* Destroy video memory name database. */ + gcmkVERIFY_OK(gckKERNEL_DestroyIntegerDatabase(Kernel, Kernel->db->nameDatabase)); + } + + if (Kernel->db->nameDatabaseMutex) + { + /* Destroy video memory name database mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->nameDatabaseMutex)); + } + + if (Kernel->db->pointerDatabase) + { + /* Destroy id-pointer database. */ + gcmkVERIFY_OK(gckKERNEL_DestroyIntegerDatabase(Kernel, Kernel->db->pointerDatabase)); + } + + if (Kernel->db->videoMemListMutex) + { + /* Destroy video memory list mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->videoMemListMutex)); + } + + /* Destroy the database. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Kernel->db)); + + /* Notify stuck timer to quit. */ + Kernel->monitorTimerStop = gcvTRUE; + } + +#if gcdDVFS + if (Kernel->dvfs) + { + gcmkVERIFY_OK(gckDVFS_Stop(Kernel->dvfs)); + gcmkVERIFY_OK(gckDVFS_Destroy(Kernel->dvfs)); + } +#endif + +#if gcdLINUX_SYNC_FILE + if (Kernel->timeline) + { + gcmkVERIFY_OK(gckOS_DestroySyncTimeline(Kernel->os, Kernel->timeline)); + } +#endif + +#if gcdSECURITY + if (Kernel->securityChannel) + { + gcmkVERIFY_OK(gckKERNEL_SecurityClose(Kernel->securityChannel)); + } +#endif + + /* Mark the gckKERNEL object as unknown. */ + Kernel->object.type = gcvOBJ_UNKNOWN; + + /* Free the gckKERNEL object. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Kernel)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckKERNEL_AllocateVideoMemory +** +** Walk requested pools to allocate video memory. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** OUTPUT: +** +** gckVIDMEM_NODE * NodeObject +** Pointer to a variable receiving video memory represetation. +*/ +gceSTATUS +gckKERNEL_AllocateVideoMemory( + IN gckKERNEL Kernel, + IN gctUINT32 Alignment, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN OUT gctSIZE_T * Bytes, + IN OUT gcePOOL * Pool, + OUT gckVIDMEM_NODE * NodeObject + ) +{ + gceSTATUS status; + gcePOOL pool; + gckVIDMEM videoMemory; + gctINT loopCount; + gckVIDMEM_NODE nodeObject = gcvNULL; + gctBOOL contiguous = gcvFALSE; + gctBOOL cacheable = gcvFALSE; + gctBOOL secure = gcvFALSE; + gctBOOL fastPools = gcvFALSE; + gctBOOL virtualPool4K = gcvFALSE; + gctBOOL hasFastPools = gcvFALSE; + gctSIZE_T bytes = *Bytes; + + gcmkHEADER_ARG("Kernel=%p *Pool=%d *Bytes=%lu Alignment=%lu Type=%d", + Kernel, *Pool, *Bytes, Alignment, Type); + + *NodeObject = gcvNULL; + + /* Check flags. */ + contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS; + cacheable = Flag & gcvALLOC_FLAG_CACHEABLE; + secure = Flag & gcvALLOC_FLAG_SECURITY; + + if (Flag & gcvALLOC_FLAG_FAST_POOLS) + { + fastPools = gcvTRUE; + Flag &= ~gcvALLOC_FLAG_FAST_POOLS; + } + + if (Flag & gcvALLOC_FLAG_4K_PAGES) + { + virtualPool4K = gcvTRUE; + Flag &= ~gcvALLOC_FLAG_4K_PAGES; + } + +#if gcdALLOC_ON_FAULT + if (Type == gcvVIDMEM_COLOR_BUFFER) + { + Flag |= gcvALLOC_FLAG_ALLOC_ON_FAULT; + } +#endif + + if (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT) + { + *Pool = gcvPOOL_VIRTUAL; + } + + if (Flag & gcvALLOC_FLAG_DMABUF_EXPORTABLE) + { + gctSIZE_T pageSize = 0; + gckOS_GetPageSize(Kernel->os, &pageSize); + + /* Usually, the exported dmabuf might be later imported to DRM, + ** while DRM requires input size to be page aligned. + */ + bytes = gcmALIGN(bytes, pageSize); + } + + if (Type == gcvVIDMEM_TYPE_COMMAND) + { +#if gcdALLOC_CMD_FROM_RESERVE || gcdSECURITY || gcdDISABLE_GPU_VIRTUAL_ADDRESS || !USE_KERNEL_VIRTUAL_BUFFERS + Flag |= gcvALLOC_FLAG_CONTIGUOUS; +#endif + } + + if (Type == gcvVIDMEM_TYPE_TILE_STATUS) + { + gctBOOL tileStatusInVirtual; + + { + tileStatusInVirtual = + gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_MC20); + } + + if (!tileStatusInVirtual) + { + /* Must be contiguous if not support virtual tile status. */ + Flag |= gcvALLOC_FLAG_CONTIGUOUS; + } + } + +AllocateMemory: + +#if gcdCAPTURE_ONLY_MODE + if (*Pool != gcvPOOL_VIRTUAL) + { + *Pool = gcvPOOL_SYSTEM; + } +#endif + + /* Get initial pool. */ + switch (pool = *Pool) + { + case gcvPOOL_DEFAULT: + case gcvPOOL_LOCAL: + pool = gcvPOOL_LOCAL_INTERNAL; + loopCount = (gctINT) gcvPOOL_NUMBER_OF_POOLS; + break; + + case gcvPOOL_UNIFIED: + pool = gcvPOOL_SYSTEM; + loopCount = (gctINT) gcvPOOL_NUMBER_OF_POOLS; + break; + + default: + loopCount = 1; + break; + } + + while (loopCount-- > 0) + { + if (pool == gcvPOOL_VIRTUAL) + { + /* Try contiguous virtual first. */ +#if gcdCONTIGUOUS_SIZE_LIMIT + if (bytes > gcdCONTIGUOUS_SIZE_LIMIT && contiguous == gcvFALSE) + { + status = gcvSTATUS_OUT_OF_MEMORY; + } + else +#endif +#if gcdENABLE_GPU_1M_PAGE + if (!virtualPool4K && Kernel->core != gcvCORE_VG && Kernel->hardware->mmuVersion) + { + /* Create a gckVIDMEM_NODE from contiguous memory. */ + status = gckVIDMEM_NODE_AllocateVirtualChunk( + Kernel, + pool, + Type, + Flag | gcvALLOC_FLAG_CONTIGUOUS, + &bytes, + &nodeObject); + + if (gcmIS_SUCCESS(status)) + { + /* Memory allocated. */ + break; + } + } +#endif + { + /* Create a gckVIDMEM_NODE from contiguous memory. */ + status = gckVIDMEM_NODE_AllocateVirtual( + Kernel, + pool, + Type, + Flag | gcvALLOC_FLAG_CONTIGUOUS, + &bytes, + &nodeObject); + } + + if (gcmIS_SUCCESS(status)) + { + /* Memory allocated. */ + break; + } + + if (contiguous) + { + break; + } + +#if gcdENABLE_GPU_1M_PAGE + /* Try non-contiguous virtual chunk. */ + if (!virtualPool4K && Kernel->hardware->mmuVersion && Kernel->core != gcvCORE_VG) + { + /* Create a gckVIDMEM_NODE from contiguous memory. */ + status = gckVIDMEM_NODE_AllocateVirtualChunk( + Kernel, + pool, + Type, + Flag | gcvALLOC_FLAG_NON_CONTIGUOUS, + &bytes, + &nodeObject); + + if (gcmIS_SUCCESS(status)) + { + /* Memory allocated. */ + break; + } + } +#endif + /* Try non-contiguous virtual. */ + /* Create a gckVIDMEM_NODE for virtual memory. */ + gcmkONERROR( + gckVIDMEM_NODE_AllocateVirtual(Kernel, + pool, + Type, + Flag | gcvALLOC_FLAG_NON_CONTIGUOUS, + &bytes, &nodeObject)); + + /* Success. */ + break; + } + /* gcvPOOL_SYSTEM/gcvPOOL_SRAM can't be cacheable. */ + else if (cacheable == gcvFALSE && secure == gcvFALSE) + { +#ifdef EMULATOR + /* Cmodel only support 1 SRAM currently. */ + Kernel->sRAMIndex = 0; + Kernel->extSRAMIndex = 0; +#endif + + /* Get pointer to gckVIDMEM object for pool. */ + status = gckKERNEL_GetVideoMemoryPool(Kernel, pool, &videoMemory); + + if (gcmIS_SUCCESS(status)) + { + /* Allocate memory. */ + if ((Flag & videoMemory->capability) != Flag) + { + status = gcvSTATUS_NOT_SUPPORTED; + + } +#if defined(gcdLINEAR_SIZE_LIMIT) + /* 512 KB */ + else if (bytes > gcdLINEAR_SIZE_LIMIT) + { + status = gcvSTATUS_OUT_OF_MEMORY; + } +#endif + else + { + hasFastPools = gcvTRUE; + status = gckVIDMEM_NODE_AllocateLinear(Kernel, + videoMemory, + pool, + Type, + Flag, + Alignment, + (pool == gcvPOOL_SYSTEM || + pool == gcvPOOL_INTERNAL_SRAM || + pool == gcvPOOL_EXTERNAL_SRAM), + &bytes, + &nodeObject); + } + + if (gcmIS_SUCCESS(status)) + { + /* Memory allocated. */ + break; + } +#if gcdCAPTURE_ONLY_MODE + else + { + gcmkPRINT("Capture only mode: Out of Memory"); + } +#endif + + } + } + + if (pool == gcvPOOL_LOCAL_INTERNAL) + { + /* Advance to external memory. */ + pool = gcvPOOL_LOCAL_EXTERNAL; + } + + else + if (pool == gcvPOOL_LOCAL_EXTERNAL) + { + if (Kernel->sRAMLoopMode) + { + /* Advance to Internal SRAM memory block. */ + pool = gcvPOOL_INTERNAL_SRAM; + } + else + { + /* Advance to contiguous reserved memory. */ + pool = gcvPOOL_SYSTEM; + } + } + + else + if (pool == gcvPOOL_INTERNAL_SRAM) + { + if (Kernel->sRAMIndex < gcvSRAM_INTER_COUNT - 1 && !Kernel->sRAMPhysFaked[Kernel->sRAMIndex]) + { + Kernel->sRAMIndex++; + loopCount++; + } + else + { + /* Advance to contiguous reserved memory. */ + pool = gcvPOOL_SYSTEM; + } + } + + else + if (pool == gcvPOOL_SYSTEM) + { + /* Do not go ahead to try relative slow pools */ + if (fastPools && hasFastPools) + { + status = gcvSTATUS_OUT_OF_MEMORY; + break; + } + + /* Advance to virtual memory. */ + pool = gcvPOOL_VIRTUAL; + } + + else + { + /* Out of pools. */ + gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY); + } + } + + if (nodeObject == gcvNULL) + { + if (contiguous) + { + /* Broadcast OOM message. */ + status = gckOS_Broadcast(Kernel->os, Kernel->hardware, gcvBROADCAST_OUT_OF_MEMORY); + + if (gcmIS_SUCCESS(status)) + { + /* Get some memory. */ + gckOS_Delay(gcvNULL, 1); + goto AllocateMemory; + } + } + + /* Nothing allocated. */ + gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY); + } + +#if gcdCAPTURE_ONLY_MODE + nodeObject->captureSize = bytes; +#endif + + /* Return node and pool used for allocation. */ + *Pool = pool; + *Bytes = bytes; + *NodeObject = nodeObject; + + /* Return status. */ + gcmkFOOTER_ARG("*Pool=%d *NodeObject=%p", *Pool, *NodeObject); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + + +/******************************************************************************* +** +** _AllocateLinearMemory +** +** Private function to allocate the requested amount of video memory, output +** video memory handle. +*/ +gceSTATUS +_AllocateLinearMemory( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + gctUINT32 handle = 0; + gceDATABASE_TYPE dbType; + gcePOOL pool = (gcePOOL)Interface->u.AllocateLinearVideoMemory.pool; + gctSIZE_T bytes = (gctSIZE_T)Interface->u.AllocateLinearVideoMemory.bytes; + gctUINT32 alignment = Interface->u.AllocateLinearVideoMemory.alignment; + gceVIDMEM_TYPE type = (Interface->u.AllocateLinearVideoMemory.type & 0xFF); + gctUINT32 flag = Interface->u.AllocateLinearVideoMemory.flag; + gctUINT64 mappingInOne = 1; + gctBOOL isContiguous; + + gcmkHEADER_ARG("Kernel=%p pool=%d bytes=%lu alignment=%lu type=%d", + Kernel, pool, bytes, alignment, type); + + gcmkVERIFY_ARGUMENT(bytes != 0); + + if (Interface->u.AllocateLinearVideoMemory.sRAMIndex >= 0) + { + Kernel->sRAMIndex = Interface->u.AllocateLinearVideoMemory.sRAMIndex; + } + + if (Interface->u.AllocateLinearVideoMemory.extSRAMIndex >= 0) + { + Kernel->extSRAMIndex = Interface->u.AllocateLinearVideoMemory.extSRAMIndex; + } + + gckOS_QueryOption(Kernel->os, "allMapInOne", &mappingInOne); + if (mappingInOne == 0) + { + /* TODO: it should page align if driver uses dynamic mapping for mapped user memory. + * it should be adjusted with different os. + */ + alignment = gcmALIGN(alignment, 4096); + } + + /* Allocate video memory node. */ + gcmkONERROR( + gckKERNEL_AllocateVideoMemory(Kernel, + alignment, + type, + flag, + &bytes, + &pool, + &nodeObject)); + + /* Allocate handle for this video memory. */ + gcmkONERROR( + gckVIDMEM_HANDLE_Allocate(Kernel, nodeObject, &handle)); + + /* Return node and pool used for allocation. */ + Interface->u.AllocateLinearVideoMemory.node = handle; + Interface->u.AllocateLinearVideoMemory.pool = pool; + Interface->u.AllocateLinearVideoMemory.bytes = bytes; + + /* Encode surface type and pool to database type. */ + dbType = gcvDB_VIDEO_MEMORY + | (type << gcdDB_VIDEO_MEMORY_TYPE_SHIFT) + | (pool << gcdDB_VIDEO_MEMORY_POOL_SHIFT); + + /* Record in process db. */ + gcmkONERROR( + gckKERNEL_AddProcessDB(Kernel, + ProcessID, + dbType, + gcmINT2PTR(handle), + gcvNULL, + bytes)); + + gcmkONERROR(gckVIDMEM_NODE_IsContiguous(Kernel, nodeObject, &isContiguous)); + + if (isContiguous) + { + /* Record in process db. */ + gcmkONERROR( + gckKERNEL_AddProcessDB(Kernel, + ProcessID, + gcvDB_CONTIGUOUS, + gcmINT2PTR(handle), + gcvNULL, + bytes)); + } + + if (type & gcvVIDMEM_TYPE_COMMAND) + { + /* Record in process db. */ + gcmkONERROR( + gckKERNEL_AddProcessDB(Kernel, + ProcessID, + gcvDB_COMMAND_BUFFER, + gcmINT2PTR(handle), + gcvNULL, + bytes)); + } + + /* Return status. */ + gcmkFOOTER_ARG("pool=%d node=0x%x", pool, handle); + return gcvSTATUS_OK; + +OnError: + if (handle) + { + /* Destroy handle allocated. */ + gcmkVERIFY_OK(gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, handle)); + } + + if (nodeObject) + { + /* Free video memory allocated. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, nodeObject)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** _ReleaseVideoMemory +** +** Release handle of a video memory. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctUINT32 ProcessID +** ProcessID of current process. +** +** gctUINT32 Handle +** Handle of video memory. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +_ReleaseVideoMemory( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 Handle + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject; + gceDATABASE_TYPE type; + gctBOOL isContiguous; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d Handle=%d", + Kernel, ProcessID, Handle); + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Kernel, ProcessID, Handle, &nodeObject)); + + type = gcvDB_VIDEO_MEMORY + | (nodeObject->type << gcdDB_VIDEO_MEMORY_TYPE_SHIFT) + | (nodeObject->pool << gcdDB_VIDEO_MEMORY_POOL_SHIFT); + + gcmkONERROR( + gckKERNEL_RemoveProcessDB(Kernel, + ProcessID, + type, + gcmINT2PTR(Handle))); + + gcmkONERROR(gckVIDMEM_NODE_IsContiguous(Kernel, nodeObject, &isContiguous)); + + if (isContiguous) + { + gckKERNEL_RemoveProcessDB(Kernel, + ProcessID, + gcvDB_CONTIGUOUS, + gcmINT2PTR(Handle)); + } + + if (nodeObject->type & gcvVIDMEM_TYPE_COMMAND) + { + gckKERNEL_RemoveProcessDB(Kernel, + ProcessID, + gcvDB_COMMAND_BUFFER, + gcmINT2PTR(Handle)); + } + + gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, Handle); + + gckVIDMEM_NODE_Dereference(Kernel, nodeObject); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** _LockVideoMemory +** +** Lock a video memory node. It will generate a cpu virtual address used +** by software and a GPU address used by GPU. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gceCORE Core +** GPU to which video memory is locked. +** +** gcsHAL_INTERFACE * Interface +** Pointer to a gcsHAL_INTERFACE structure that defines the command to +** be dispatched. +** +** OUTPUT: +** +** gcsHAL_INTERFACE * Interface +** Pointer to a gcsHAL_INTERFACE structure that receives any data to be +** returned. +*/ +static gceSTATUS +_LockVideoMemory( + IN gckKERNEL Kernel, + IN gceCORE Core, + IN gctUINT32 ProcessID, + IN OUT gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gctUINT32 handle; + gckVIDMEM_NODE nodeObject = gcvNULL; + gctBOOL referenced = gcvFALSE; + gctUINT32 address = gcvINVALID_ADDRESS; + gctPOINTER logical = gcvNULL; + gctPHYS_ADDR_T physical = gcvINVALID_PHYSICAL_ADDRESS; + gctUINT32 gid = 0; + gctBOOL asynchronous = gcvFALSE; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d", + Kernel, ProcessID); + + handle = Interface->u.LockVideoMemory.node; + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Kernel, ProcessID, handle, &nodeObject)); + + /* Ref node. */ + gcmkONERROR(gckVIDMEM_NODE_Reference(Kernel, nodeObject)); + referenced = gcvTRUE; + +#if gcdCAPTURE_ONLY_MODE + if (Interface->u.LockVideoMemory.queryCapSize) + { + Interface->u.LockVideoMemory.captureSize = nodeObject->captureSize; + return gcvSTATUS_OK; + } + else + { + nodeObject->captureLogical = Interface->u.LockVideoMemory.captureLogical; + } +#endif + + /* Lock for userspace CPU userspace. */ + gcmkONERROR( + gckVIDMEM_NODE_LockCPU(Kernel, + nodeObject, + Interface->u.LockVideoMemory.cacheable, + gcvTRUE, + &logical)); + + /* Lock for GPU address. */ + gcmkONERROR(gckVIDMEM_NODE_Lock(Kernel, nodeObject, &address)); + + /* Get CPU physical address. */ + gcmkONERROR(gckVIDMEM_NODE_GetPhysical(Kernel, nodeObject, 0, &physical)); + gcmkONERROR(gckVIDMEM_NODE_GetGid(Kernel, nodeObject, &gid)); + + Interface->u.LockVideoMemory.address = address; + Interface->u.LockVideoMemory.memory = gcmPTR_TO_UINT64(logical); + Interface->u.LockVideoMemory.physicalAddress = physical; + Interface->u.LockVideoMemory.gid = gid; + + gcmkONERROR( + gckKERNEL_AddProcessDB(Kernel, + ProcessID, + gcvDB_VIDEO_MEMORY_LOCKED, + gcmINT2PTR(handle), + logical, + 0)); + + /* Ref handle. */ + gckVIDMEM_HANDLE_Reference(Kernel, ProcessID, handle); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (logical) + { + gckVIDMEM_NODE_UnlockCPU(Kernel, nodeObject, ProcessID, gcvTRUE, gcvFALSE); + } + + if (address) + { + gckVIDMEM_NODE_Unlock(Kernel, nodeObject, ProcessID, &asynchronous); + + if (asynchronous) + { + gckVIDMEM_NODE_Unlock(Kernel, nodeObject, ProcessID, gcvNULL); + } + } + + if (referenced) + { + gckVIDMEM_NODE_Dereference(Kernel, nodeObject); + } + + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** _UnlockVideoMemory +** +** Unlock a video memory node. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctUINT32 ProcessID +** ProcessID of current process. +** +** gcsHAL_INTERFACE * Interface +** Pointer to a gcsHAL_INTERFACE structure that defines the command to +** be dispatched. +** +** OUTPUT: +** +** gcsHAL_INTERFACE * Interface +** Pointer to a gcsHAL_INTERFACE structure that receives any data to be +** returned. +*/ +static gceSTATUS +_UnlockVideoMemory( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN OUT gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject; + gcuVIDMEM_NODE_PTR node; + gckVIDMEM_BLOCK vidMemBlock = gcvNULL; + gctSIZE_T bytes; + gctUINT64 mappingInOne = 1; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d", + Kernel, ProcessID); + + Interface->u.UnlockVideoMemory.pool = gcvPOOL_UNKNOWN; + Interface->u.UnlockVideoMemory.bytes = 0; + + gcmkONERROR(gckVIDMEM_HANDLE_Lookup( + Kernel, + ProcessID, + (gctUINT32)Interface->u.UnlockVideoMemory.node, + &nodeObject + )); + + gckOS_QueryOption(Kernel->os, "allMapInOne", &mappingInOne); + /* Unlock CPU. */ + gcmkONERROR(gckVIDMEM_NODE_UnlockCPU( + Kernel, nodeObject, ProcessID, gcvTRUE, mappingInOne == 1)); + + /* Unlock video memory. */ + gcmkONERROR(gckVIDMEM_NODE_Unlock( + Kernel, + nodeObject, + ProcessID, + &Interface->u.UnlockVideoMemory.asynchroneous + )); + + /* Leave deref handle and deref node in later operation. */ + + node = nodeObject->node; + + vidMemBlock = node->VirtualChunk.parent; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + bytes = node->VidMem.bytes; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + bytes = node->VirtualChunk.bytes; + } + else + { + bytes = node->Virtual.bytes; + } + + Interface->u.UnlockVideoMemory.pool = nodeObject->pool; + Interface->u.UnlockVideoMemory.bytes = bytes; + +#if gcdCAPTURE_ONLY_MODE + Interface->u.UnlockVideoMemory.captureLogical = nodeObject->captureLogical; +#endif + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* + * Unlikely to fail expect error node or unlocked, there's no error roll + * back requried for those two conditions. + */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** _BottomHalfUnlockVideoMemory +** +** Unlock video memory from gpu. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctUINT32 ProcessID +** Process ID owning this memory. +** +** gceVIDMEM_TYPE +** Video memory allocation type. +** +** gctPOINTER Pointer +** Video memory to be unlock. +*/ +static gceSTATUS +_BottomHalfUnlockVideoMemory( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Node + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + + /* Remove record from process db. */ + gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB( + Kernel, + ProcessID, + gcvDB_VIDEO_MEMORY_LOCKED, + gcmINT2PTR(Node) + )); + + gcmkONERROR(gckVIDMEM_HANDLE_Lookup( + Kernel, + ProcessID, + Node, + &nodeObject + )); + + /* Deref handle. */ + gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, Node); + + /* Unlock video memory, synced. */ + gcmkONERROR(gckVIDMEM_NODE_Unlock(Kernel, nodeObject, ProcessID, gcvNULL)); + + /* Deref node. */ + gcmkONERROR(gckVIDMEM_NODE_Dereference(Kernel, nodeObject)); + + return gcvSTATUS_OK; + +OnError: + return status; +} + +static gceSTATUS +_WrapUserMemory( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + gceDATABASE_TYPE type; + gctUINT32 handle = 0; + + gcmkONERROR( + gckVIDMEM_NODE_WrapUserMemory(Kernel, + &Interface->u.WrapUserMemory.desc, + Interface->u.WrapUserMemory.type, + &nodeObject, + &Interface->u.WrapUserMemory.bytes)); + + /* Create handle representation for userspace. */ + gcmkONERROR( + gckVIDMEM_HANDLE_Allocate(Kernel, + nodeObject, + &handle)); + + type = gcvDB_VIDEO_MEMORY + | (nodeObject->type << gcdDB_VIDEO_MEMORY_TYPE_SHIFT) + | (nodeObject->pool << gcdDB_VIDEO_MEMORY_POOL_SHIFT); + + gcmkONERROR( + gckKERNEL_AddProcessDB(Kernel, + ProcessID, + type, + gcmINT2PTR(handle), + gcvNULL, + (gctSIZE_T)Interface->u.WrapUserMemory.bytes)); + + Interface->u.WrapUserMemory.node = handle; + return gcvSTATUS_OK; + +OnError: + if (handle) + { + gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, handle); + } + + if (nodeObject) + { + gckVIDMEM_NODE_Dereference(Kernel, nodeObject); + } + + return status; +} + +static gceSTATUS +_ExportVideoMemory( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Kernel, + ProcessID, + Interface->u.ExportVideoMemory.node, + &nodeObject)); + + gcmkONERROR( + gckVIDMEM_NODE_Export(Kernel, + nodeObject, + Interface->u.ExportVideoMemory.flags, + gcvNULL, + &Interface->u.ExportVideoMemory.fd)); + +OnError: + return status; +} + +static gceSTATUS +_NameVideoMemory( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Kernel, + ProcessID, + Interface->u.NameVideoMemory.handle, + &nodeObject)); + + gcmkONERROR( + gckVIDMEM_NODE_Name(Kernel, + nodeObject, + &Interface->u.NameVideoMemory.name)); +OnError: + return status; +} + +static gceSTATUS +_ImportVideoMemory( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + gctUINT32 handle = 0; + + gcmkONERROR( + gckVIDMEM_NODE_Import(Kernel, + Interface->u.ImportVideoMemory.name, + &nodeObject)); + + /* Create handle representation for userspace. */ + gcmkONERROR( + gckVIDMEM_HANDLE_Allocate(Kernel, + nodeObject, + &handle)); + + gcmkONERROR( + gckKERNEL_AddProcessDB(Kernel, + ProcessID, + gcvDB_VIDEO_MEMORY, + gcmINT2PTR(handle), + gcvNULL, + 0)); + + Interface->u.ImportVideoMemory.handle = handle; + return gcvSTATUS_OK; + +OnError: + if (handle) + { + gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, handle); + } + + if (nodeObject) + { + gckVIDMEM_NODE_Dereference(Kernel, nodeObject); + } + + return status; +} + +/******************************************************************************* +** +** gckKERNEL_SetVidMemMetadata +** +** Set/Get metadata to/from gckVIDMEM_NODE object. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctUINT32 ProcessID +** ProcessID of current process. +** +** INOUT: +** +** gcsHAL_INTERFACE * Interface +** Pointer to a interface structure +*/ +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include <linux/dma-buf.h> + +gceSTATUS +_SetVidMemMetadata( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + INOUT gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status = gcvSTATUS_NOT_SUPPORTED; + gckVIDMEM_NODE nodeObj = gcvNULL; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d", Kernel, ProcessID); + + gcmkONERROR(gckVIDMEM_HANDLE_Lookup(Kernel, ProcessID, Interface->u.SetVidMemMetadata.node, &nodeObj)); + + if (Interface->u.SetVidMemMetadata.readback) + { + Interface->u.SetVidMemMetadata.ts_fd = nodeObj->metadata.ts_fd; + Interface->u.SetVidMemMetadata.fc_enabled = nodeObj->metadata.fc_enabled; + Interface->u.SetVidMemMetadata.fc_value = nodeObj->metadata.fc_value; + Interface->u.SetVidMemMetadata.fc_value_upper = nodeObj->metadata.fc_value_upper; + Interface->u.SetVidMemMetadata.compressed = nodeObj->metadata.compressed; + Interface->u.SetVidMemMetadata.compress_format = nodeObj->metadata.compress_format; + } + else + { +#ifdef gcdANDROID + if (nodeObj->metadata.ts_address == 0 && nodeObj->tsNode != NULL) + { + gctUINT32 PhysicalAddress = 0; + + /* Lock for GPU address. */ + gcmkONERROR(gckVIDMEM_NODE_Lock(Kernel, nodeObj->tsNode, &PhysicalAddress)); + + nodeObj->metadata.ts_address = ( + PhysicalAddress + Kernel->hardware->baseAddress); + + gcmkONERROR(gckVIDMEM_NODE_Unlock(Kernel, nodeObj->tsNode, ProcessID, gcvNULL)); + } +#else + nodeObj->metadata.ts_fd = Interface->u.SetVidMemMetadata.ts_fd; + + if (nodeObj->metadata.ts_fd >= 0) + { + nodeObj->metadata.ts_dma_buf = dma_buf_get(nodeObj->metadata.ts_fd); + + if (IS_ERR(nodeObj->metadata.ts_dma_buf)) + { + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + + dma_buf_put(nodeObj->metadata.ts_dma_buf); + } + else + { + nodeObj->metadata.ts_dma_buf = NULL; + } +#endif + + nodeObj->metadata.fc_enabled = Interface->u.SetVidMemMetadata.fc_enabled; + nodeObj->metadata.fc_value = Interface->u.SetVidMemMetadata.fc_value; + nodeObj->metadata.fc_value_upper = Interface->u.SetVidMemMetadata.fc_value_upper; + nodeObj->metadata.compressed = Interface->u.SetVidMemMetadata.compressed; + nodeObj->metadata.compress_format = Interface->u.SetVidMemMetadata.compress_format; + } + +OnError: + gcmkFOOTER(); + return status; +} + +#else + +gceSTATUS +_SetVidMemMetadata( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + INOUT gcsHAL_INTERFACE * Interface + ) +{ + gcmkFATAL("The kernel did NOT support CONFIG_DMA_SHARED_BUFFER"); + return gcvSTATUS_NOT_SUPPORTED; +} +#endif + +static gceSTATUS +_GetVideoMemoryFd( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Kernel, + ProcessID, + Interface->u.GetVideoMemoryFd.handle, + &nodeObject)); + + gcmkONERROR( + gckVIDMEM_NODE_GetFd(Kernel, + nodeObject, + &Interface->u.GetVideoMemoryFd.fd)); + + /* No need to add it to processDB because OS will release all fds when + ** process quits. + */ +OnError: + return status; +} + + +gceSTATUS +gckKERNEL_QueryDatabase( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN OUT gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gctINT i; + + gceDATABASE_TYPE type[2] = { + gcvDB_VIDEO_MEMORY | (gcvPOOL_SYSTEM << gcdDB_VIDEO_MEMORY_POOL_SHIFT), + gcvDB_VIDEO_MEMORY | (gcvPOOL_VIRTUAL << gcdDB_VIDEO_MEMORY_POOL_SHIFT), + }; + + gcmkHEADER(); + + /* Query video memory. */ + gcmkONERROR( + gckKERNEL_QueryProcessDB(Kernel, + Interface->u.Database.processID, + !Interface->u.Database.validProcessID, + gcvDB_VIDEO_MEMORY, + &Interface->u.Database.vidMem)); + + /* Query non-paged memory. */ + gcmkONERROR( + gckKERNEL_QueryProcessDB(Kernel, + Interface->u.Database.processID, + !Interface->u.Database.validProcessID, + gcvDB_NON_PAGED, + &Interface->u.Database.nonPaged)); + + /* Query contiguous memory. */ + gcmkONERROR( + gckKERNEL_QueryProcessDB(Kernel, + Interface->u.Database.processID, + !Interface->u.Database.validProcessID, + gcvDB_CONTIGUOUS, + &Interface->u.Database.contiguous)); + + /* Query GPU idle time. */ + gcmkONERROR( + gckKERNEL_QueryProcessDB(Kernel, + Interface->u.Database.processID, + !Interface->u.Database.validProcessID, + gcvDB_IDLE, + &Interface->u.Database.gpuIdle)); + for (i = 0; i < 2; i++) + { + /* Query each video memory pool. */ + gcmkONERROR( + gckKERNEL_QueryProcessDB(Kernel, + Interface->u.Database.processID, + !Interface->u.Database.validProcessID, + type[i], + &Interface->u.Database.vidMemPool[i])); + } + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gckKERNEL_DumpVidMemUsage(Kernel, Interface->u.Database.processID); +#endif + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_ConfigPowerManagement( + IN gckKERNEL Kernel, + IN OUT gcsHAL_INTERFACE * Interface +) +{ + gceSTATUS status; + gctBOOL enable = Interface->u.ConfigPowerManagement.enable; + + gcmkHEADER(); + + gcmkONERROR(gckHARDWARE_EnablePowerManagement(Kernel->hardware, enable)); + + if (enable == gcvFALSE) + { + gcmkONERROR( + gckHARDWARE_SetPowerState(Kernel->hardware, gcvPOWER_ON)); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + + +static gceSTATUS +gckKERNEL_CacheOperation( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 Node, + IN gceCACHEOPERATION Operation, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + gcuVIDMEM_NODE_PTR node = gcvNULL; + gckVIDMEM_BLOCK vidMemBlock = gcvNULL; + gctSIZE_T offset = 0; + void *memHandle; + + gcmkHEADER_ARG("Kernel=%p pid=%u Node=%u op=%d Logical=%p Bytes=0x%lx", + Kernel, ProcessID, Node, Operation, Logical, Bytes); + + gcmkONERROR(gckVIDMEM_HANDLE_Lookup(Kernel, + ProcessID, + Node, + &nodeObject)); + + node = nodeObject->node; + + vidMemBlock = node->VirtualChunk.parent; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + static gctBOOL printed; + + if (!printed) + { + printed = gcvTRUE; + gcmkPRINT("[galcore]: %s: Flush Video Memory", __FUNCTION__); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + memHandle = vidMemBlock->physical; + offset = node->VirtualChunk.offset; + } + else + { + memHandle = node->Virtual.physical; + } + + switch (Operation) + { + case gcvCACHE_FLUSH: + /* Clean and invalidate the cache. */ + status = gckOS_CacheFlush(Kernel->os, + ProcessID, + memHandle, + offset, + Logical, + Bytes); + break; + case gcvCACHE_CLEAN: + /* Clean the cache. */ + status = gckOS_CacheClean(Kernel->os, + ProcessID, + memHandle, + offset, + Logical, + Bytes); + break; + case gcvCACHE_INVALIDATE: + /* Invalidate the cache. */ + status = gckOS_CacheInvalidate(Kernel->os, + ProcessID, + memHandle, + offset, + Logical, + Bytes); + break; + + case gcvCACHE_MEMORY_BARRIER: + status = gckOS_MemoryBarrier(Kernel->os, Logical); + break; + + default: + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + break; + } + +OnError: + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_WaitFence( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE node; + gckCOMMAND command = Kernel->command; + gckCOMMAND asyncCommand = Kernel->asyncCommand; + gckFENCE fence = gcvNULL; + gctUINT i; + + gcmkASSERT(command != gcvNULL); + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Kernel, + ProcessID, + Interface->u.WaitFence.handle, + &node)); + + gcmkONERROR(gckVIDMEM_NODE_Reference(Kernel, node)); + + /* Wait for fence of all engines. */ + for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) + { + gckFENCE_SYNC sync = &node->sync[i]; + + if (i == gcvENGINE_RENDER) + { + fence = command->fence; + } + else + { + fence = asyncCommand->fence; + } + + gcmkONERROR(gckVIDMEM_NODE_InvalidateCache( + Kernel, + fence->videoMem, + 0, + fence->logical, + 8 + )); + + if (sync->commitStamp <= *(gctUINT64_PTR)fence->logical) + { + continue; + } + else + { + gckOS_Signal(Kernel->os, sync->signal, gcvFALSE); + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, &fence->mutex, gcvINFINITE)); + + /* Add to waiting list. */ + gcsLIST_AddTail(&sync->head, &fence->waitingList); + + gcmkASSERT(sync->inList == gcvFALSE); + + sync->inList = gcvTRUE; + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, &fence->mutex)); + + /* Wait. */ + status = gckOS_WaitSignal( + Kernel->os, + sync->signal, + gcvTRUE, + Interface->u.WaitFence.timeOut + ); + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, &fence->mutex, gcvINFINITE)); + + if (sync->inList) + { + gcsLIST_Del(&sync->head); + sync->inList = gcvFALSE; + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, &fence->mutex)); + } + } + + gckVIDMEM_NODE_Dereference(Kernel, node); + +OnError: + return status; +} + +static gceSTATUS +_Commit( + IN gckDEVICE Device, + IN gceHARDWARE_TYPE HwType, + IN gceENGINE Engine, + IN gctUINT32 ProcessId, + IN OUT gcsHAL_COMMIT * Commit + ) +{ + gceSTATUS status; + gcsHAL_SUBCOMMIT *subCommit = &Commit->subCommit; + gcsHAL_SUBCOMMIT _subCommit; + gctPOINTER userPtr = gcvNULL; + gctBOOL needCopy = gcvFALSE; + gckKERNEL kernel; + + gcmkVERIFY_OK(gckOS_QueryNeedCopy(Device->os, ProcessId, &needCopy)); + + do + { + gckCOMMAND command; + gckEVENT eventObj; + gctUINT64 next; + + /* Skip the first nested sub-commit struct. */ + if (userPtr) + { + /* Copy/map sub-commit from user. */ + if (needCopy) + { + subCommit = &_subCommit; + + status = gckOS_CopyFromUserData( + Device->os, + subCommit, + userPtr, + gcmSIZEOF(gcsHAL_SUBCOMMIT) + ); + } + else + { + status = gckOS_MapUserPointer( + Device->os, + userPtr, + gcmSIZEOF(gcsHAL_SUBCOMMIT), + (gctPOINTER *)&subCommit + ); + } + + if (gcmIS_ERROR(status)) + { + userPtr = gcvNULL; + + gcmkONERROR(status); + } + } + + if (subCommit->coreId >= gcvCORE_COUNT) + { + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + /* Determine the objects. */ + if (HwType == gcvHARDWARE_3D || HwType == gcvHARDWARE_3D2D || HwType == gcvHARDWARE_VIP) + { + kernel = Device->coreInfoArray[subCommit->coreId].kernel; + } + else + { + kernel = Device->map[HwType].kernels[subCommit->coreId]; + } + if (Engine == gcvENGINE_BLT) + { + command = kernel->asyncCommand; + eventObj = kernel->asyncEvent; + } + else + { + command = kernel->command; + eventObj = kernel->eventObj; + } + + { + /* Commit command buffers. */ + status = gckCOMMAND_Commit(command, + subCommit, + ProcessId, + Commit->shared, + &Commit->commitStamp, + &Commit->contextSwitched); + + if (status != gcvSTATUS_INTERRUPTED) + { + gcmkONERROR(status); + } + + /* Commit events. */ + status = gckEVENT_Commit( + eventObj, + gcmUINT64_TO_PTR(subCommit->queue), + kernel->hardware->options.powerManagement + ); + + if (status != gcvSTATUS_INTERRUPTED) + { + gcmkONERROR(status); + } + } + + next = subCommit->next; + + /* Unmap user pointer if mapped. */ + if (!needCopy && userPtr) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Device->os, + userPtr, + gcmSIZEOF(gcsHAL_SUBCOMMIT), + subCommit + )); + } + + /* Advance to next sub-commit from user. */ + userPtr = gcmUINT64_TO_PTR(next); + } + while (userPtr); + + subCommit = &Commit->subCommit; + userPtr = gcvNULL; + + if (HwType == gcvHARDWARE_3D || HwType == gcvHARDWARE_3D2D || HwType == gcvHARDWARE_VIP) + { + kernel = Device->coreInfoArray[subCommit->coreId].kernel; + } + else + { + kernel = Device->map[HwType].kernels[subCommit->coreId]; + } + + if (!kernel->hardware->options.gpuProfiler || !kernel->profileEnable) + { + return gcvSTATUS_OK; + } + + do + { + gctUINT64 next; + + /* Skip the first nested sub-commit struct. */ + if (userPtr) + { + /* Copy/map sub-commit from user. */ + if (needCopy) + { + subCommit = &_subCommit; + + status = gckOS_CopyFromUserData( + Device->os, + subCommit, + userPtr, + gcmSIZEOF(gcsHAL_SUBCOMMIT) + ); + } + else + { + status = gckOS_MapUserPointer( + Device->os, + userPtr, + gcmSIZEOF(gcsHAL_SUBCOMMIT), + (gctPOINTER *)&subCommit + ); + } + + if (gcmIS_ERROR(status)) + { + userPtr = gcvNULL; + + gcmkONERROR(status); + } + } + + if (HwType == gcvHARDWARE_3D || HwType == gcvHARDWARE_3D2D || HwType == gcvHARDWARE_VIP) + { + kernel = Device->coreInfoArray[subCommit->coreId].kernel; + } + else + { + kernel = Device->map[HwType].kernels[subCommit->coreId]; + } + + if ((kernel->hardware->options.gpuProfiler == gcvTRUE) && + (kernel->profileEnable == gcvTRUE)) + { + gcmkONERROR(gckCOMMAND_Stall(kernel->command, gcvTRUE)); + + if (kernel->command->currContext) + { + gcmkONERROR(gckHARDWARE_UpdateContextProfile( + kernel->hardware, + kernel->command->currContext)); + } + } + + next = subCommit->next; + + /* Unmap user pointer if mapped. */ + if (!needCopy && userPtr) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Device->os, + userPtr, + gcmSIZEOF(gcsHAL_SUBCOMMIT), + subCommit + )); + } + + /* Advance to next sub-commit from user. */ + userPtr = gcmUINT64_TO_PTR(next); + } + while (userPtr); + + return gcvSTATUS_OK; + +OnError: + if (!needCopy && userPtr) + { + gckOS_UnmapUserPointer( + Device->os, + userPtr, + gcmSIZEOF(gcsHAL_SUBCOMMIT), + subCommit + ); + } + + return status; +} + +#ifdef __linux__ +typedef struct _gcsGRRAPHIC_BUFFER_PARCLE +{ + gcsFDPRIVATE base; + gckKERNEL kernel; + + gckVIDMEM_NODE node[3]; + gctSHBUF shBuf; + gctINT32 signal; +} +gcsGRAPHIC_BUFFER_PARCLE; + +static void +_ReleaseGraphicBuffer( + gckKERNEL Kernel, + gcsGRAPHIC_BUFFER_PARCLE * Parcle + ) +{ + gctUINT i; + + for (i = 0; i < 3; i++) + { + if (Parcle->node[i]) + { + gckVIDMEM_NODE_Dereference(Kernel, Parcle->node[i]); + } + } + + if (Parcle->shBuf) + { + gckKERNEL_DestroyShBuffer(Kernel, Parcle->shBuf); + } + + if (Parcle->signal) + { + gckOS_DestroyUserSignal(Kernel->os, Parcle->signal); + } + + gcmkOS_SAFE_FREE(Kernel->os, Parcle); +} + +static gctINT +_FdReleaseGraphicBuffer( + gcsFDPRIVATE_PTR Private + ) +{ + gcsGRAPHIC_BUFFER_PARCLE * parcle = (gcsGRAPHIC_BUFFER_PARCLE *) Private; + + _ReleaseGraphicBuffer(parcle->kernel, parcle); + return 0; +} + +static gceSTATUS +_GetGraphicBufferFd( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 Node[3], + IN gctUINT64 ShBuf, + IN gctUINT64 Signal, + OUT gctINT32 * Fd + ) +{ + gceSTATUS status; + gctUINT i; + gcsGRAPHIC_BUFFER_PARCLE * parcle = gcvNULL; + + gcmkONERROR(gckOS_Allocate( + Kernel->os, + gcmSIZEOF(gcsGRAPHIC_BUFFER_PARCLE), + (gctPOINTER *)&parcle + )); + + gckOS_ZeroMemory(parcle, sizeof(gcsGRAPHIC_BUFFER_PARCLE)); + + parcle->base.release = _FdReleaseGraphicBuffer; + parcle->kernel = Kernel; + + for (i = 0; i < 3 && Node[i] != 0; i++) + { + gckVIDMEM_NODE nodeObject = gcvNULL; + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Kernel, ProcessID, Node[i], &nodeObject)); + + gcmkONERROR(gckVIDMEM_NODE_Reference(Kernel, nodeObject)); + + parcle->node[i] = nodeObject; + } + + if (ShBuf) + { + gctSHBUF shBuf = gcmUINT64_TO_PTR(ShBuf); + + gcmkONERROR(gckKERNEL_MapShBuffer(Kernel, shBuf)); + parcle->shBuf = shBuf; + } + + if (Signal) + { + gctSIGNAL signal = gcmUINT64_TO_PTR(Signal); + + gcmkONERROR( + gckOS_MapSignal(Kernel->os, + signal, + (gctHANDLE)(gctUINTPTR_T)ProcessID, + &signal)); + + parcle->signal= (gctINT32)Signal; + } + + gcmkONERROR(gckOS_GetFd("viv-gr", &parcle->base, Fd)); + + return gcvSTATUS_OK; + +OnError: + if (parcle) + { + _ReleaseGraphicBuffer(Kernel, parcle); + } + return status; +} +#endif + +/******************************************************************************* +** +** gckKERNEL_Dispatch +** +** Dispatch a command received from the user HAL layer. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gcsHAL_INTERFACE * Interface +** Pointer to a gcsHAL_INTERFACE structure that defines the command to +** be dispatched. +** +** OUTPUT: +** +** gcsHAL_INTERFACE * Interface +** Pointer to a gcsHAL_INTERFACE structure that receives any data to be +** returned. +*/ +gceSTATUS +gckKERNEL_Dispatch( + IN gckKERNEL Kernel, + IN gckDEVICE Device, + IN OUT gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctPHYS_ADDR physical = gcvNULL; + gctSIZE_T bytes; + gctPOINTER logical = gcvNULL; +#if (gcdENABLE_3D) + gckCONTEXT context = gcvNULL; +#endif + gckKERNEL kernel = Kernel; + gctUINT32 processID; +#if !USE_NEW_LINUX_SIGNAL + gctSIGNAL signal; +#endif + + gctBOOL powerMutexAcquired = gcvFALSE; + gctBOOL commitMutexAcquired = gcvFALSE; + gctBOOL idle = gcvFALSE; + + gcmkHEADER_ARG("Kernel=%p Interface=%p", Kernel, Interface); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Interface != gcvNULL); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL, + "Dispatching command %d (%s)", + Interface->command, _DispatchText[Interface->command]); + + gcmSTATIC_ASSERT(gcvHAL_DESTROY_MMU == gcmCOUNTOF(_DispatchText) - 1, + "DispatchText array does not match command codes"); +#endif +#if QNX_SINGLE_THREADED_DEBUGGING + gckOS_AcquireMutex(Kernel->os, Kernel->debugMutex, gcvINFINITE); +#endif + + /* Get the current process ID. */ + gcmkONERROR(gckOS_GetProcessID(&processID)); + + /* Dispatch on command. */ + switch (Interface->command) + { + case gcvHAL_GET_BASE_ADDRESS: + /* Get base address. */ + Interface->u.GetBaseAddress.baseAddress = Kernel->hardware->baseAddress; + Interface->u.GetBaseAddress.flatMappingRangeCount = Kernel->mmu->gpuPhysicalRangeCount; + if (Kernel->mmu->gpuPhysicalRangeCount) + { + gckOS_MemCopy(Interface->u.GetBaseAddress.flatMappingRanges, Kernel->mmu->gpuPhysicalRanges, + gcmSIZEOF(gcsFLAT_MAPPING_RANGE) * Kernel->mmu->gpuPhysicalRangeCount); + } + break; + + case gcvHAL_QUERY_VIDEO_MEMORY: + /* Query video memory size. */ + gcmkONERROR(gckKERNEL_QueryVideoMemory(Kernel, Interface)); + break; + + case gcvHAL_QUERY_CHIP_IDENTITY: + /* Query chip identity. */ + gcmkONERROR( + gckHARDWARE_QueryChipIdentity( + Kernel->hardware, + &Interface->u.QueryChipIdentity)); + break; + + case gcvHAL_QUERY_CHIP_FREQUENCY: + /* Query chip clock. */ + gcmkONERROR( + gckHARDWARE_QueryFrequency(Kernel->hardware)); + + Interface->u.QueryChipFrequency.mcClk = Kernel->hardware->mcClk; + Interface->u.QueryChipFrequency.shClk = Kernel->hardware->shClk; + break; + + case gcvHAL_MAP_MEMORY: + physical = gcmINT2PTR(Interface->u.MapMemory.physName); + + /* Map memory. */ + gcmkONERROR( + gckKERNEL_MapMemory(Kernel, + physical, + (gctSIZE_T) Interface->u.MapMemory.bytes, + &logical)); + + Interface->u.MapMemory.logical = gcmPTR_TO_UINT64(logical); + + gcmkVERIFY_OK( + gckKERNEL_AddProcessDB(Kernel, + processID, gcvDB_MAP_MEMORY, + logical, + physical, + (gctSIZE_T) Interface->u.MapMemory.bytes)); + break; + + case gcvHAL_UNMAP_MEMORY: + physical = gcmINT2PTR(Interface->u.UnmapMemory.physName); + + gcmkVERIFY_OK( + gckKERNEL_RemoveProcessDB(Kernel, + processID, gcvDB_MAP_MEMORY, + gcmUINT64_TO_PTR(Interface->u.UnmapMemory.logical))); + + /* Unmap memory. */ + gcmkONERROR( + gckKERNEL_UnmapMemory(Kernel, + physical, + (gctSIZE_T) Interface->u.UnmapMemory.bytes, + gcmUINT64_TO_PTR(Interface->u.UnmapMemory.logical), + processID)); + break; + + case gcvHAL_ALLOCATE_NON_PAGED_MEMORY: + bytes = (gctSIZE_T) Interface->u.AllocateNonPagedMemory.bytes; + + /* Allocate non-paged memory. */ + gcmkONERROR( + gckOS_AllocateNonPagedMemory( + Kernel->os, + gcvTRUE, + gcvALLOC_FLAG_CONTIGUOUS, + &bytes, + &physical, + &logical)); + + Interface->u.AllocateNonPagedMemory.bytes = bytes; + Interface->u.AllocateNonPagedMemory.logical = gcmPTR_TO_UINT64(logical); + Interface->u.AllocateNonPagedMemory.physName = gcmPTR_TO_NAME(physical); + + gcmkVERIFY_OK( + gckKERNEL_AddProcessDB(Kernel, + processID, gcvDB_NON_PAGED, + logical, + gcmINT2PTR(Interface->u.AllocateNonPagedMemory.physName), + bytes)); + break; + + case gcvHAL_FREE_NON_PAGED_MEMORY: + physical = gcmNAME_TO_PTR(Interface->u.FreeNonPagedMemory.physName); + + gcmkVERIFY_OK( + gckKERNEL_RemoveProcessDB(Kernel, + processID, gcvDB_NON_PAGED, + gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical))); + + /* Free non-paged memory. */ + gcmkONERROR( + gckOS_FreeNonPagedMemory(Kernel->os, + physical, + gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical), + (gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes)); + + gcmRELEASE_NAME(Interface->u.FreeNonPagedMemory.physName); + break; + + case gcvHAL_ALLOCATE_LINEAR_VIDEO_MEMORY: + /* Allocate memory. */ + gcmkONERROR(_AllocateLinearMemory(Kernel, processID, Interface)); + break; + + case gcvHAL_RELEASE_VIDEO_MEMORY: + /* Release video memory. */ + gcmkONERROR(_ReleaseVideoMemory( + Kernel, processID, + (gctUINT32)Interface->u.ReleaseVideoMemory.node + )); + break; + + case gcvHAL_LOCK_VIDEO_MEMORY: + /* Lock video memory. */ + gcmkONERROR(_LockVideoMemory(Kernel, Kernel->core, processID, Interface)); + break; + + case gcvHAL_UNLOCK_VIDEO_MEMORY: + /* Unlock video memory. */ + gcmkONERROR(_UnlockVideoMemory(Kernel, processID, Interface)); + break; + + case gcvHAL_BOTTOM_HALF_UNLOCK_VIDEO_MEMORY: + gcmkERR_BREAK(_BottomHalfUnlockVideoMemory(Kernel, processID, + Interface->u.BottomHalfUnlockVideoMemory.type, + Interface->u.BottomHalfUnlockVideoMemory.node)); + break; + + case gcvHAL_EVENT_COMMIT: + if (!Interface->commitMutex) + { + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, + Kernel->device->commitMutex, + gcvINFINITE + )); + + commitMutexAcquired = gcvTRUE; + } + /* Commit an event queue. */ + if (Interface->engine == gcvENGINE_BLT) + { + if (!gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_ASYNC_BLIT)) + { + gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); + } + + gcmkONERROR(gckEVENT_Commit( + Kernel->asyncEvent, gcmUINT64_TO_PTR(Interface->u.Event.queue), gcvFALSE)); + } + else + { + gcmkONERROR(gckEVENT_Commit( + Kernel->eventObj, gcmUINT64_TO_PTR(Interface->u.Event.queue), gcvFALSE)); + } + + if (!Interface->commitMutex) + { + gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->device->commitMutex)); + commitMutexAcquired = gcvFALSE; + } + break; + + case gcvHAL_COMMIT: + if (!Interface->commitMutex) + { + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, + Device->commitMutex, + gcvINFINITE + )); + commitMutexAcquired = gcvTRUE; + } + + gcmkONERROR(_Commit(Device, + Kernel->hardware->type, + Interface->engine, + processID, + &Interface->u.Commit)); + + if (!Interface->commitMutex) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Device->commitMutex)); + commitMutexAcquired = gcvFALSE; + } + break; + +#if !USE_NEW_LINUX_SIGNAL + case gcvHAL_USER_SIGNAL: + /* Dispatch depends on the user signal subcommands. */ + switch(Interface->u.UserSignal.command) + { + case gcvUSER_SIGNAL_CREATE: + /* Create a signal used in the user space. */ + gcmkONERROR( + gckOS_CreateUserSignal(Kernel->os, + Interface->u.UserSignal.manualReset, + &Interface->u.UserSignal.id)); + + gcmkVERIFY_OK( + gckKERNEL_AddProcessDB(Kernel, + processID, gcvDB_SIGNAL, + gcmINT2PTR(Interface->u.UserSignal.id), + gcvNULL, + 0)); + break; + + case gcvUSER_SIGNAL_DESTROY: + gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB( + Kernel, + processID, gcvDB_SIGNAL, + gcmINT2PTR(Interface->u.UserSignal.id))); + + /* Destroy the signal. */ + gcmkONERROR( + gckOS_DestroyUserSignal(Kernel->os, + Interface->u.UserSignal.id)); + break; + + case gcvUSER_SIGNAL_SIGNAL: + /* Signal the signal. */ + gcmkONERROR( + gckOS_SignalUserSignal(Kernel->os, + Interface->u.UserSignal.id, + Interface->u.UserSignal.state)); + break; + + case gcvUSER_SIGNAL_WAIT: + /* Wait on the signal. */ + status = gckOS_WaitUserSignal(Kernel->os, + Interface->u.UserSignal.id, + Interface->u.UserSignal.wait); + break; + + case gcvUSER_SIGNAL_MAP: + gcmkONERROR( + gckOS_MapSignal(Kernel->os, + (gctSIGNAL)(gctUINTPTR_T)Interface->u.UserSignal.id, + (gctHANDLE)(gctUINTPTR_T)processID, + &signal)); + + gcmkVERIFY_OK( + gckKERNEL_AddProcessDB(Kernel, + processID, gcvDB_SIGNAL, + gcmINT2PTR(Interface->u.UserSignal.id), + gcvNULL, + 0)); + break; + + case gcvUSER_SIGNAL_UNMAP: + gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB( + Kernel, + processID, gcvDB_SIGNAL, + gcmINT2PTR(Interface->u.UserSignal.id))); + + /* Destroy the signal. */ + gcmkONERROR( + gckOS_DestroyUserSignal(Kernel->os, + Interface->u.UserSignal.id)); + break; + + default: + /* Invalid user signal command. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + break; +#endif + + case gcvHAL_SET_POWER_MANAGEMENT_STATE: + /* Set the power management state. */ + gcmkONERROR( + gckHARDWARE_SetPowerState(Kernel->hardware, + Interface->u.SetPowerManagement.state)); + break; + + case gcvHAL_QUERY_POWER_MANAGEMENT_STATE: + /* Chip is not idle. */ + Interface->u.QueryPowerManagement.isIdle = gcvFALSE; + + /* Query the power management state. */ + gcmkONERROR(gckHARDWARE_QueryPowerState( + Kernel->hardware, + &Interface->u.QueryPowerManagement.state)); + + /* Query the idle state. */ + gcmkONERROR( + gckHARDWARE_QueryIdle(Kernel->hardware, + &Interface->u.QueryPowerManagement.isIdle)); + break; + + case gcvHAL_READ_REGISTER: +#if gcdREGISTER_READ_FROM_USER + { + gceCHIPPOWERSTATE power; + + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->hardware->powerMutex, gcvINFINITE)); + powerMutexAcquired = gcvTRUE; + gcmkONERROR(gckHARDWARE_QueryPowerState(Kernel->hardware, + &power)); + if (power == gcvPOWER_ON) + { + /* Read a register. */ + gcmkONERROR(gckOS_ReadRegisterEx( + Kernel->os, + Kernel->core, + Interface->u.ReadRegisterData.address, + &Interface->u.ReadRegisterData.data)); + } + else + { + /* Chip is in power-state. */ + Interface->u.ReadRegisterData.data = 0; + status = gcvSTATUS_CHIP_NOT_READY; + } + gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex)); + powerMutexAcquired = gcvFALSE; + } +#else + /* No access from user land to read registers. */ + Interface->u.ReadRegisterData.data = 0; + status = gcvSTATUS_NOT_SUPPORTED; +#endif + break; + + case gcvHAL_WRITE_REGISTER: +#if gcdREGISTER_WRITE_FROM_USER + { + gceCHIPPOWERSTATE power; + + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->hardware->powerMutex, gcvINFINITE)); + powerMutexAcquired = gcvTRUE; + gcmkONERROR(gckHARDWARE_QueryPowerState(Kernel->hardware, + &power)); + if (power == gcvPOWER_ON) + { + /* Write a register. */ + gcmkONERROR( + gckOS_WriteRegisterEx(Kernel->os, + Kernel->core, + Interface->u.WriteRegisterData.address, + Interface->u.WriteRegisterData.data)); + } + else + { + /* Chip is in power-state. */ + Interface->u.WriteRegisterData.data = 0; + status = gcvSTATUS_CHIP_NOT_READY; + } + gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex)); + powerMutexAcquired = gcvFALSE; + } +#else + /* No access from user land to write registers. */ + status = gcvSTATUS_NOT_SUPPORTED; +#endif + break; + + case gcvHAL_READ_ALL_PROFILE_REGISTERS_PART1: + /* Read profile data according to the context. */ + gcmkONERROR( + gckHARDWARE_QueryContextProfile( + Kernel->hardware, + Kernel->profileCleanRegister, + gcmNAME_TO_PTR(Interface->u.RegisterProfileData_part1.context), + &Interface->u.RegisterProfileData_part1.Counters, + gcvNULL)); + break; + case gcvHAL_READ_ALL_PROFILE_REGISTERS_PART2: + /* Read profile data according to the context. */ + gcmkONERROR( + gckHARDWARE_QueryContextProfile( + Kernel->hardware, + Kernel->profileCleanRegister, + gcmNAME_TO_PTR(Interface->u.RegisterProfileData_part2.context), + gcvNULL, + &Interface->u.RegisterProfileData_part2.Counters)); + break; + + case gcvHAL_GET_PROFILE_SETTING: +#if VIVANTE_PROFILER + /* Get profile setting */ + Interface->u.GetProfileSetting.enable = Kernel->profileEnable; +#endif + + status = gcvSTATUS_OK; + break; + + case gcvHAL_SET_PROFILE_SETTING: +#if VIVANTE_PROFILER + /* Set profile setting */ + if(Kernel->hardware->options.gpuProfiler) + { + Kernel->profileEnable = Interface->u.SetProfileSetting.enable; + + if (Kernel->profileEnable) + { + gcmkONERROR(gckHARDWARE_InitProfiler(Kernel->hardware)); + } + + } + else + { + status = gcvSTATUS_NOT_SUPPORTED; + break; + } +#endif + + status = gcvSTATUS_OK; + break; + + case gcvHAL_READ_PROFILER_REGISTER_SETTING: + Kernel->profileCleanRegister = Interface->u.SetProfilerRegisterClear.bclear; + status = gcvSTATUS_OK; + break; + + case gcvHAL_RESET: + /* Reset the hardware. */ + gcmkONERROR( + gckHARDWARE_Reset(Kernel->hardware)); + break; + + case gcvHAL_SET_DEBUG_LEVEL_ZONE: + /* Set debug level and zones. */ + gckOS_SetDebugLevel(Interface->u.DebugLevelZone.level); + gckOS_SetDebugZones(Interface->u.DebugLevelZone.zones, + Interface->u.DebugLevelZone.enable); + break; + + case gcvHAL_DEBUG_DUMP: + gckOS_DumpBuffer(Kernel->os, + Interface->u.DebugDump.type, + gcmUINT64_TO_PTR(Interface->u.DebugDump.ptr), + Interface->u.DebugDump.address, + Interface->u.DebugDump.size); + status = gcvSTATUS_OK; + break; + + case gcvHAL_DUMP_GPU_STATE: + { + gceCHIPPOWERSTATE power; + + _DumpDriverConfigure(Kernel); + + gcmkONERROR(gckHARDWARE_QueryPowerState( + Kernel->hardware, + &power + )); + + if (power == gcvPOWER_ON) + { + Interface->u.ReadRegisterData.data = 1; + + _DumpState(Kernel); + } + else + { + Interface->u.ReadRegisterData.data = 0; + status = gcvSTATUS_CHIP_NOT_READY; + + gcmkPRINT("[galcore]: Can't dump state if GPU isn't POWER ON."); + } + } + break; + + case gcvHAL_DUMP_EVENT: + break; + + case gcvHAL_CACHE: + logical = gcmUINT64_TO_PTR(Interface->u.Cache.logical); + bytes = (gctSIZE_T) Interface->u.Cache.bytes; + + gcmkONERROR(gckKERNEL_CacheOperation(Kernel, + processID, + Interface->u.Cache.node, + Interface->u.Cache.operation, + logical, + bytes)); + break; + + case gcvHAL_TIMESTAMP: + /* Check for invalid timer. */ + if ((Interface->u.TimeStamp.timer >= gcmCOUNTOF(Kernel->timers)) + || (Interface->u.TimeStamp.request != 2)) + { + Interface->u.TimeStamp.timeDelta = 0; + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + /* Return timer results and reset timer. */ + { + gcsTIMER_PTR timer = &(Kernel->timers[Interface->u.TimeStamp.timer]); + gctUINT64 timeDelta = 0; + + if (timer->stopTime < timer->startTime ) + { + Interface->u.TimeStamp.timeDelta = 0; + gcmkONERROR(gcvSTATUS_TIMER_OVERFLOW); + } + + timeDelta = timer->stopTime - timer->startTime; + + /* Check truncation overflow. */ + Interface->u.TimeStamp.timeDelta = (gctINT32) timeDelta; + /*bit0~bit30 is available*/ + if (timeDelta>>31) + { + Interface->u.TimeStamp.timeDelta = 0; + gcmkONERROR(gcvSTATUS_TIMER_OVERFLOW); + } + + status = gcvSTATUS_OK; + } + break; + + case gcvHAL_DATABASE: + gcmkONERROR(gckKERNEL_QueryDatabase(Kernel, processID, Interface)); + break; + + case gcvHAL_VERSION: + Interface->u.Version.major = gcvVERSION_MAJOR; + Interface->u.Version.minor = gcvVERSION_MINOR; + Interface->u.Version.patch = gcvVERSION_PATCH; + Interface->u.Version.build = gcvVERSION_BUILD; +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL, + "KERNEL version %s", gcvVERSION_STRING); +#endif + break; + + case gcvHAL_CHIP_INFO: + /* Only if not support multi-core */ + Interface->u.ChipInfo.count = 1; + Interface->u.ChipInfo.types[0] = Kernel->hardware->type; + break; + +#if (gcdENABLE_3D) + case gcvHAL_ATTACH: + if (Kernel->command) + { +#if gcdCAPTURE_ONLY_MODE + gckVIDMEM_NODE nodeObject = gcvNULL; + + if (Interface->u.Attach.queryCapSize) + { + /* Attach user process. */ + gcmkONERROR( + gckCOMMAND_Attach(Kernel->command, + &context, + &bytes, + &Interface->u.Attach.numStates, + processID)); + + Interface->u.Attach.maxState = bytes; + Interface->u.Attach.context = gcmPTR_TO_NAME(context); + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Kernel, processID, context->buffer->handle, &nodeObject)); + + Interface->u.Attach.captureSize = nodeObject->captureSize; + break; + } + else + { + gctUINT i = 0; + context = gcmNAME_TO_PTR(Interface->u.Attach.context); + + for (i = 0; i < gcdCONTEXT_BUFFER_NUM; ++i) + { + gcsCONTEXT_PTR buffer = context->buffer; + + gckOS_CopyToUserData(Kernel->os, buffer->logical, Interface->u.Attach.contextLogical[i], Interface->u.Attach.captureSize); + + buffer = buffer->next; + } + } + +#else + /* Attach user process. */ + gcmkONERROR( + gckCOMMAND_Attach(Kernel->command, + &context, + &bytes, + &Interface->u.Attach.numStates, + processID)); + + Interface->u.Attach.maxState = bytes; + Interface->u.Attach.context = gcmPTR_TO_NAME(context); +#endif + + if (Interface->u.Attach.map) + { + if (context != gcvNULL) + { +#if gcdCAPTURE_ONLY_MODE + gctUINT i = 0; + + for (i = 0; i < gcdCONTEXT_BUFFER_NUM; ++i) + { + Interface->u.Attach.logicals[i] = gcmPTR_TO_UINT64(Interface->u.Attach.contextLogical[i]); + } + + Interface->u.Attach.bytes = (gctUINT)context->totalSize; +#else + gcmkVERIFY_OK( + gckCONTEXT_MapBuffer(context, + Interface->u.Attach.logicals, + &Interface->u.Attach.bytes)); +#endif + } + else + { + gctUINT i; + + for (i = 0; i < gcmCOUNTOF(Interface->u.Attach.logicals); i++) + { + Interface->u.Attach.logicals[i] = 0; + } + + Interface->u.Attach.bytes = 0; + } + } + + gcmkVERIFY_OK( + gckKERNEL_AddProcessDB(Kernel, + processID, gcvDB_CONTEXT, + gcmINT2PTR(Interface->u.Attach.context), + gcvNULL, + 0)); + } + break; +#endif + + case gcvHAL_DETACH: + if (Kernel->command) + { + gcmkVERIFY_OK( + gckKERNEL_RemoveProcessDB(Kernel, + processID, gcvDB_CONTEXT, + gcmINT2PTR(Interface->u.Detach.context))); + + /* Detach user process. */ + gcmkONERROR( + gckCOMMAND_Detach(Kernel->command, + gcmNAME_TO_PTR(Interface->u.Detach.context))); + + gcmRELEASE_NAME(Interface->u.Detach.context); + } + break; + + case gcvHAL_GET_FRAME_INFO: + gcmkONERROR(gckHARDWARE_GetFrameInfo( + Kernel->hardware, + gcmUINT64_TO_PTR(Interface->u.GetFrameInfo.frameInfo))); + break; + + case gcvHAL_DUMP_GPU_PROFILE: + gcmkONERROR(gckHARDWARE_DumpGpuProfile(Kernel->hardware)); + break; + + case gcvHAL_SET_FSCALE_VALUE: +#if gcdENABLE_FSCALE_VAL_ADJUST + /* Wait for HW idle, otherwise it is not safe. */ + gcmkONERROR(gckCOMMAND_Stall(Kernel->command, gcvFALSE)); + + for (;;) + { + gcmkONERROR(gckHARDWARE_QueryIdle(Kernel->hardware, &idle)); + + if (idle) + { + break; + } + + gcmkVERIFY_OK(gckOS_Delay(Kernel->os, 1)); + } + + status = gckHARDWARE_SetFscaleValue(Kernel->hardware, + Interface->u.SetFscaleValue.value, + Interface->u.SetFscaleValue.shValue); +#else + status = gcvSTATUS_NOT_SUPPORTED; +#endif + break; + case gcvHAL_GET_FSCALE_VALUE: +#if gcdENABLE_FSCALE_VAL_ADJUST + status = gckHARDWARE_GetFscaleValue(Kernel->hardware, + &Interface->u.GetFscaleValue.value, + &Interface->u.GetFscaleValue.minValue, + &Interface->u.GetFscaleValue.maxValue); +#else + status = gcvSTATUS_NOT_SUPPORTED; +#endif + break; + + case gcvHAL_EXPORT_VIDEO_MEMORY: + /* Unlock video memory. */ + gcmkONERROR(_ExportVideoMemory(Kernel, processID, Interface)); + break; + + case gcvHAL_NAME_VIDEO_MEMORY: + gcmkONERROR(_NameVideoMemory(Kernel, processID, Interface)); + break; + + case gcvHAL_IMPORT_VIDEO_MEMORY: + gcmkONERROR(_ImportVideoMemory(Kernel, processID, Interface)); + break; + + case gcvHAL_SET_VIDEO_MEMORY_METADATA: + gcmkONERROR(_SetVidMemMetadata(Kernel, processID, Interface)); + break; + + case gcvHAL_GET_VIDEO_MEMORY_FD: + gcmkONERROR(_GetVideoMemoryFd(Kernel, processID, Interface)); + break; + + case gcvHAL_QUERY_RESET_TIME_STAMP: + Interface->u.QueryResetTimeStamp.timeStamp = Kernel->resetTimeStamp; + Interface->u.QueryResetTimeStamp.contextID = Kernel->hardware->contextID; + break; + +#if gcdLINUX_SYNC_FILE + case gcvHAL_CREATE_NATIVE_FENCE: + { + gctINT fenceFD; + gctSIGNAL signal = + gcmUINT64_TO_PTR(Interface->u.CreateNativeFence.signal); + + gcmkONERROR( + gckOS_CreateNativeFence(Kernel->os, + Kernel->timeline, + signal, + &fenceFD)); + + Interface->u.CreateNativeFence.fenceFD = fenceFD; + } + break; + + case gcvHAL_WAIT_NATIVE_FENCE: + { + gctINT fenceFD; + gctUINT32 timeout; + + fenceFD = Interface->u.WaitNativeFence.fenceFD; + timeout = Interface->u.WaitNativeFence.timeout; + + gcmkONERROR( + gckOS_WaitNativeFence(Kernel->os, + Kernel->timeline, + fenceFD, + timeout)); + } + break; +#endif + + case gcvHAL_SHBUF: + { + gctSHBUF shBuf; + gctPOINTER uData; + gctUINT32 bytes; + + switch (Interface->u.ShBuf.command) + { + case gcvSHBUF_CREATE: + bytes = Interface->u.ShBuf.bytes; + + /* Create. */ + gcmkONERROR(gckKERNEL_CreateShBuffer(Kernel, bytes, &shBuf)); + + Interface->u.ShBuf.id = gcmPTR_TO_UINT64(shBuf); + + gcmkVERIFY_OK( + gckKERNEL_AddProcessDB(Kernel, + processID, + gcvDB_SHBUF, + shBuf, + gcvNULL, + 0)); + break; + + case gcvSHBUF_DESTROY: + shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id); + + /* Check db first to avoid illegal destroy in the process. */ + gcmkONERROR( + gckKERNEL_RemoveProcessDB(Kernel, + processID, + gcvDB_SHBUF, + shBuf)); + + gcmkONERROR(gckKERNEL_DestroyShBuffer(Kernel, shBuf)); + break; + + case gcvSHBUF_MAP: + shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id); + + /* Map for current process access. */ + gcmkONERROR(gckKERNEL_MapShBuffer(Kernel, shBuf)); + + gcmkVERIFY_OK( + gckKERNEL_AddProcessDB(Kernel, + processID, + gcvDB_SHBUF, + shBuf, + gcvNULL, + 0)); + break; + + case gcvSHBUF_WRITE: + shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id); + uData = gcmUINT64_TO_PTR(Interface->u.ShBuf.data); + bytes = Interface->u.ShBuf.bytes; + + /* Write. */ + gcmkONERROR( + gckKERNEL_WriteShBuffer(Kernel, shBuf, uData, bytes)); + break; + + case gcvSHBUF_READ: + shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id); + uData = gcmUINT64_TO_PTR(Interface->u.ShBuf.data); + bytes = Interface->u.ShBuf.bytes; + + /* Read. */ + gcmkONERROR( + gckKERNEL_ReadShBuffer(Kernel, + shBuf, + uData, + bytes, + &bytes)); + + /* Return copied size. */ + Interface->u.ShBuf.bytes = bytes; + break; + + default: + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + break; + } + } + break; + +#ifdef __linux__ + case gcvHAL_GET_GRAPHIC_BUFFER_FD: + gcmkONERROR(_GetGraphicBufferFd( + Kernel, + processID, + Interface->u.GetGraphicBufferFd.node, + Interface->u.GetGraphicBufferFd.shBuf, + Interface->u.GetGraphicBufferFd.signal, + &Interface->u.GetGraphicBufferFd.fd + )); + break; +#endif + + + case gcvHAL_CONFIG_POWER_MANAGEMENT: + gcmkONERROR(gckKERNEL_ConfigPowerManagement(Kernel, Interface)); + break; + + case gcvHAL_WRAP_USER_MEMORY: + gcmkONERROR(_WrapUserMemory(Kernel, processID, Interface)); + break; + + case gcvHAL_WAIT_FENCE: + gcmkONERROR(_WaitFence(Kernel, processID, Interface)); + break; + + case gcvHAL_DEVICE_MUTEX: + if (Interface->u.DeviceMutex.isMutexLocked) + { + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, + Kernel->device->commitMutex, + gcvINFINITE + )); + } + else + { + gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->device->commitMutex)); + } + break; + +#if gcdDEC_ENABLE_AHB + case gcvHAL_DEC300_READ: + gcmkONERROR(viv_dec300_read( + Interface->u.DEC300Read.enable, + Interface->u.DEC300Read.readId, + Interface->u.DEC300Read.format, + Interface->u.DEC300Read.strides, + Interface->u.DEC300Read.is3D, + Interface->u.DEC300Read.isMSAA, + Interface->u.DEC300Read.clearValue, + Interface->u.DEC300Read.isTPC, + Interface->u.DEC300Read.isTPCCompressed, + Interface->u.DEC300Read.surfAddrs, + Interface->u.DEC300Read.tileAddrs + )); + break; + + case gcvHAL_DEC300_WRITE: + gcmkONERROR(viv_dec300_write( + Interface->u.DEC300Write.enable, + Interface->u.DEC300Write.readId, + Interface->u.DEC300Write.writeId, + Interface->u.DEC300Write.format, + Interface->u.DEC300Write.surfAddr, + Interface->u.DEC300Write.tileAddr + )); + break; + + case gcvHAL_DEC300_FLUSH: + gcmkONERROR(viv_dec300_flush(0)); + break; + + case gcvHAL_DEC300_FLUSH_WAIT: + gcmkONERROR(viv_dec300_flush_done(&Interface->u.DEC300FlushWait.done)); + break; +#endif + + + case gcvHAL_QUERY_CHIP_OPTION: + /* Query chip options. */ + gcmkONERROR( + gckHARDWARE_QueryChipOptions( + Kernel->hardware, + &Interface->u.QueryChipOptions)); + break; + + default: + /* Invalid command. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + +OnError: + /* Save status. */ + Interface->status = status; + +#if QNX_SINGLE_THREADED_DEBUGGING + gckOS_ReleaseMutex(Kernel->os, Kernel->debugMutex); +#endif + + if (powerMutexAcquired == gcvTRUE) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex)); + } + + if (commitMutexAcquired == gcvTRUE) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->device->commitMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_AttachProcess +** +** Attach or detach a process. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctBOOL Attach +** gcvTRUE if a new process gets attached or gcFALSE when a process +** gets detatched. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_AttachProcess( + IN gckKERNEL Kernel, + IN gctBOOL Attach + ) +{ + gceSTATUS status; + gctUINT32 processID; + + gcmkHEADER_ARG("Kernel=%p Attach=%d", Kernel, Attach); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + + /* Get current process ID. */ + gcmkONERROR(gckOS_GetProcessID(&processID)); + + gcmkONERROR(gckKERNEL_AttachProcessEx(Kernel, Attach, processID)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_AttachProcessEx +** +** Attach or detach a process with the given PID. Can be paired with gckKERNEL_AttachProcess +** provided the programmer is aware of the consequences. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctBOOL Attach +** gcvTRUE if a new process gets attached or gcFALSE when a process +** gets detatched. +** +** gctUINT32 PID +** PID of the process to attach or detach. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_AttachProcessEx( + IN gckKERNEL Kernel, + IN gctBOOL Attach, + IN gctUINT32 PID + ) +{ + gceSTATUS status; + gctINT32 old; + + gcmkHEADER_ARG("Kernel=%p Attach=%d PID=%d", Kernel, Attach, PID); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + + if (Attach) + { + /* Increment the number of clients attached. */ + gcmkONERROR( + gckOS_AtomIncrement(Kernel->os, Kernel->atomClients, &old)); + + if (old == 0) + { + { + gcmkONERROR(gckOS_Broadcast(Kernel->os, + Kernel->hardware, + gcvBROADCAST_FIRST_PROCESS)); + } + } + + if (Kernel->dbCreated) + { + /* Create the process database. */ + gcmkONERROR(gckKERNEL_CreateProcessDB(Kernel, PID)); + } + } + else + { + if (Kernel->dbCreated) + { + /* Clean up the process database. */ + gcmkONERROR(gckKERNEL_DestroyProcessDB(Kernel, PID)); + + /* Save the last know process ID. */ + Kernel->db->lastProcessID = PID; + } + + { + status = gckEVENT_Submit(Kernel->eventObj, gcvTRUE, gcvFALSE); + + if (status == gcvSTATUS_INTERRUPTED && Kernel->eventObj->submitTimer) + { + gcmkONERROR(gckOS_StartTimer(Kernel->os, + Kernel->eventObj->submitTimer, + 1)); + } + else + { + gcmkONERROR(status); + } + } + + /* Decrement the number of clients attached. */ + gcmkONERROR( + gckOS_AtomDecrement(Kernel->os, Kernel->atomClients, &old)); + + if (old == 1) + { + { + /* Last client detached, switch to SUSPEND power state. */ + gcmkONERROR(gckOS_Broadcast(Kernel->os, + Kernel->hardware, + gcvBROADCAST_LAST_PROCESS)); + } + } + + if (Kernel->timeoutPID == PID) + { + Kernel->timeOut = Kernel->hardware->type == gcvHARDWARE_2D + ? gcdGPU_2D_TIMEOUT + : gcdGPU_TIMEOUT; + } + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckKERNEL_Recovery +** +** Try to recover the GPU from a fatal error. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_Recovery( + IN gckKERNEL Kernel + ) +{ + gceSTATUS status; + gckEVENT eventObj; + gckHARDWARE hardware; + gctUINT32 mask = 0; + gctUINT32 i = 0, count = 0; +#if gcdINTERRUPT_STATISTIC + gctINT32 oldValue; +#endif + + gcmkHEADER_ARG("Kernel=%p", Kernel); + + /* Validate the arguemnts. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + + /* Grab gckEVENT object. */ + eventObj = Kernel->eventObj; + gcmkVERIFY_OBJECT(eventObj, gcvOBJ_EVENT); + + /* Grab gckHARDWARE object. */ + hardware = Kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + if (Kernel->stuckDump == gcvSTUCK_DUMP_NONE) + { + gcmkPRINT("[galcore]: GPU[%d] hang, automatic recovery.", Kernel->core); + } + else + { + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, Kernel->device->stuckDumpMutex, gcvINFINITE)); + + _DumpDriverConfigure(Kernel); + _DumpState(Kernel); + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->device->stuckDumpMutex)); + } + + if (Kernel->recovery == gcvFALSE) + { + gcmkPRINT("[galcore]: Stop driver to keep scene."); + + /* Stop monitor timer. */ + Kernel->monitorTimerStop = gcvTRUE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + /* Issuing a soft reset for the GPU. */ + gcmkONERROR(gckHARDWARE_Reset(hardware)); + + mask = Kernel->restoreMask; + + for (i = 0; i < 32; i++) + { + if (mask & (1 << i)) + { + count++; + } + } + + /* Handle all outstanding events now. */ + gcmkONERROR(gckOS_AtomSet(Kernel->os, eventObj->pending, mask)); + +#if gcdINTERRUPT_STATISTIC + while (count--) + { + gcmkONERROR(gckOS_AtomDecrement( + Kernel->os, + eventObj->interruptCount, + &oldValue + )); + } + + gckOS_AtomClearMask(Kernel->hardware->pendingEvent, mask); +#endif + + gcmkONERROR(gckEVENT_Notify(eventObj, 1)); + + gcmkVERIFY_OK(gckOS_GetTime(&Kernel->resetTimeStamp)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckKERNEL_OpenUserData +** +** Get access to the user data. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctBOOL NeedCopy +** The flag indicating whether or not the data should be copied. +** +** gctPOINTER StaticStorage +** Pointer to the kernel storage where the data is to be copied if +** NeedCopy is gcvTRUE. +** +** gctPOINTER UserPointer +** User pointer to the data. +** +** gctSIZE_T Size +** Size of the data. +** +** OUTPUT: +** +** gctPOINTER * KernelPointer +** Pointer to the kernel pointer that will be pointing to the data. +*/ +gceSTATUS +gckKERNEL_OpenUserData( + IN gckKERNEL Kernel, + IN gctBOOL NeedCopy, + IN gctPOINTER StaticStorage, + IN gctPOINTER UserPointer, + IN gctSIZE_T Size, + OUT gctPOINTER * KernelPointer + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG( + "Kernel=%p NeedCopy=%d StaticStorage=%p " + "UserPointer=%p Size=%lu KernelPointer=%p", + Kernel, NeedCopy, StaticStorage, UserPointer, Size, KernelPointer + ); + + /* Validate the arguemnts. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(!NeedCopy || (StaticStorage != gcvNULL)); + gcmkVERIFY_ARGUMENT(UserPointer != gcvNULL); + gcmkVERIFY_ARGUMENT(KernelPointer != gcvNULL); + gcmkVERIFY_ARGUMENT(Size > 0); + + if (NeedCopy) + { + /* Copy the user data to the static storage. */ + gcmkONERROR(gckOS_CopyFromUserData( + Kernel->os, StaticStorage, UserPointer, Size + )); + + /* Set the kernel pointer. */ + * KernelPointer = StaticStorage; + } + else + { + gctPOINTER pointer = gcvNULL; + + /* Map the user pointer. */ + gcmkONERROR(gckOS_MapUserPointer( + Kernel->os, UserPointer, Size, &pointer + )); + + /* Set the kernel pointer. */ + * KernelPointer = pointer; + } + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckKERNEL_CloseUserData +** +** Release resources associated with the user data connection opened by +** gckKERNEL_OpenUserData. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctBOOL NeedCopy +** The flag indicating whether or not the data should be copied. +** +** gctBOOL FlushData +** If gcvTRUE, the data is written back to the user. +** +** gctPOINTER UserPointer +** User pointer to the data. +** +** gctSIZE_T Size +** Size of the data. +** +** OUTPUT: +** +** gctPOINTER * KernelPointer +** Kernel pointer to the data. +*/ +gceSTATUS +gckKERNEL_CloseUserData( + IN gckKERNEL Kernel, + IN gctBOOL NeedCopy, + IN gctBOOL FlushData, + IN gctPOINTER UserPointer, + IN gctSIZE_T Size, + OUT gctPOINTER * KernelPointer + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctPOINTER pointer; + + gcmkHEADER_ARG( + "Kernel=%p NeedCopy=%d FlushData=%d " + "UserPointer=%p Size=%lu KernelPointer=%p", + Kernel, NeedCopy, FlushData, UserPointer, Size, KernelPointer + ); + + /* Validate the arguemnts. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(UserPointer != gcvNULL); + gcmkVERIFY_ARGUMENT(KernelPointer != gcvNULL); + gcmkVERIFY_ARGUMENT(Size > 0); + + /* Get a shortcut to the kernel pointer. */ + pointer = * KernelPointer; + + if (pointer != gcvNULL) + { + if (NeedCopy) + { + if (FlushData) + { + gcmkONERROR(gckOS_CopyToUserData( + Kernel->os, * KernelPointer, UserPointer, Size + )); + } + } + else + { + /* Unmap record from kernel memory. */ + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Kernel->os, + UserPointer, + Size, + * KernelPointer + )); + } + + /* Reset the kernel pointer. */ + * KernelPointer = gcvNULL; + } + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static void +gckQUEUE_Dequeue( + IN gckQUEUE LinkQueue + ) +{ + gcmkASSERT(LinkQueue->count == LinkQueue->size); + + LinkQueue->count--; + LinkQueue->front = (LinkQueue->front + 1) % gcdLINK_QUEUE_SIZE; +} + +void +gckQUEUE_Enqueue( + IN gckQUEUE LinkQueue, + IN gcuQUEUEDATA *Data + ) +{ + gcuQUEUEDATA * datas = LinkQueue->datas; + + if (LinkQueue->count == LinkQueue->size) + { + gckQUEUE_Dequeue(LinkQueue); + } + + gcmkASSERT(LinkQueue->count < LinkQueue->size); + + LinkQueue->count++; + + datas[LinkQueue->rear] = *Data; + + LinkQueue->rear = (LinkQueue->rear + 1) % LinkQueue->size; +} + +void +gckQUEUE_GetData( + IN gckQUEUE LinkQueue, + IN gctUINT32 Index, + OUT gcuQUEUEDATA ** Data + ) +{ + gcuQUEUEDATA * datas = LinkQueue->datas; + + gcmkASSERT(Index < LinkQueue->size); + + *Data = &datas[(Index + LinkQueue->front) % LinkQueue->size]; +} + +gceSTATUS +gckQUEUE_Allocate( + IN gckOS Os, + IN gckQUEUE Queue, + IN gctUINT32 Size + ) +{ + gceSTATUS status; + + gcmkONERROR(gckOS_Allocate( + Os, + gcmSIZEOF(struct _gckLINKDATA) * Size, + (gctPOINTER *)&Queue->datas + )); + + Queue->size = Size; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +gceSTATUS +gckQUEUE_Free( + IN gckOS Os, + IN gckQUEUE Queue + ) +{ + if (Queue->datas) + { + gcmkVERIFY_OK(gckOS_Free(Os, (gctPOINTER)Queue->datas)); + } + + return gcvSTATUS_OK; +} + +/******************************************************************************\ +*************************** Pointer - ID translation *************************** +\******************************************************************************/ + +/* The capacity is 32 initially, double when expand, but no more than 512. */ +#define gcdID_TABLE_MAX_EXPAND 512 + +typedef struct _gcsINTEGERDB * gckINTEGERDB; +typedef struct _gcsINTEGERDB +{ + gckOS os; + gctPOINTER * table; + gctUINT32 * bitmap; + gctPOINTER mutex; + gctUINT32 capacity; + gctUINT32 nextId; + gctUINT32 freeCount; +} +gcsINTEGERDB; + +gceSTATUS +gckKERNEL_CreateIntegerDatabase( + IN gckKERNEL Kernel, + IN gctUINT32 Capacity, + OUT gctPOINTER * Database + ) +{ + gceSTATUS status; + gckINTEGERDB database = gcvNULL; + + gcmkHEADER_ARG("Kernel=%p Capacity=%u Datbase=%p", Kernel, Capacity, Database); + + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Database != gcvNULL); + + /* round up to 32 alignment. */ + Capacity = (Capacity + 31) & ~31; + + /* Allocate a database. */ + gcmkONERROR(gckOS_Allocate( + Kernel->os, gcmSIZEOF(gcsINTEGERDB), (gctPOINTER *)&database)); + + gcmkONERROR(gckOS_ZeroMemory(database, gcmSIZEOF(gcsINTEGERDB))); + + /* Allocate a pointer table. */ + gcmkONERROR(gckOS_Allocate( + Kernel->os, + gcmSIZEOF(gctPOINTER) * Capacity, + (gctPOINTER *)&database->table + )); + + /* Allocate bitmap. */ + gcmkONERROR(gckOS_Allocate( + Kernel->os, Capacity / 8, (gctPOINTER *)&database->bitmap)); + + gcmkONERROR(gckOS_ZeroMemory(database->bitmap, Capacity / 8)); + + /* Allocate a database mutex. */ + gcmkONERROR(gckOS_CreateMutex(Kernel->os, &database->mutex)); + + /* Initialize. */ + database->os = Kernel->os; + + database->nextId = 0; + database->freeCount = Capacity; + database->capacity = Capacity; + + *Database = database; + + gcmkFOOTER_ARG("*Database=0x%08X", *Database); + return gcvSTATUS_OK; + +OnError: + /* Rollback. */ + if (database) + { + if (database->table) + { + gckOS_Free(Kernel->os, database->table); + } + + if (database->bitmap) + { + gckOS_Free(Kernel->os, database->bitmap); + } + + gckOS_Free(Kernel->os, database); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_DestroyIntegerDatabase( + IN gckKERNEL Kernel, + IN gctPOINTER Database + ) +{ + gckINTEGERDB database = Database; + + gcmkHEADER_ARG("Kernel=%p Datbase=%p", Kernel, Database); + + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Database != gcvNULL); + + /* Destroy pointer table. */ + gcmkOS_SAFE_FREE(Kernel->os, database->table); + gcmkOS_SAFE_FREE(Kernel->os, database->bitmap); + + /* Destroy database mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, database->mutex)); + + /* Destroy database. */ + gckOS_Free(Kernel->os, database); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckKERNEL_AllocateIntegerId( + IN gctPOINTER Database, + IN gctPOINTER Pointer, + OUT gctUINT32 * Id + ) +{ + gceSTATUS status; + gckINTEGERDB database = Database; + gctUINT32 pos; + gctUINT32 n, i; + gckOS os = database->os; + gctPOINTER * table = gcvNULL; + + gcmkHEADER_ARG("Database=%p Pointer=%p", Database, Pointer); + + gcmkVERIFY_OK(gckOS_AcquireMutex(os, database->mutex, gcvINFINITE)); + + if (database->freeCount < 1) + { + gctUINT32 * bitmap = gcvNULL; + gctUINT32 expand; + gctUINT32 capacity; + + expand = database->capacity < gcdID_TABLE_MAX_EXPAND + ? database->capacity : gcdID_TABLE_MAX_EXPAND; + + capacity = database->capacity + expand; + + /* Extend table. */ + gcmkONERROR( + gckOS_Allocate(os, + gcmSIZEOF(gctPOINTER) * capacity, + (gctPOINTER *)&table)); + + gcmkONERROR( + gckOS_Allocate(os, capacity / 32 * sizeof(gctUINT32), (gctPOINTER *)&bitmap)); + + gckOS_ZeroMemory(bitmap + database->capacity / 32, expand / 8); + + /* Copy data from old table. */ + gckOS_MemCopy(table, + database->table, + database->capacity * gcmSIZEOF(gctPOINTER)); + + gckOS_MemCopy(bitmap, + database->bitmap, + database->capacity / 32 * sizeof(gctUINT32)); + + gckOS_Free(os, database->table); + gckOS_Free(os, database->bitmap); + + /* Update databse with new allocated table. */ + database->table = table; + database->bitmap = bitmap; + database->nextId = database->capacity; + database->capacity = capacity; + database->freeCount += expand; + } + + pos = database->nextId; + n = pos >> 5; + i = pos & 31; + + while (database->bitmap[n] & (1u << i)) + { + pos++; + + if (pos == database->capacity) + { + /* Wrap to the begin. */ + pos = 0; + } + + n = pos >> 5; + i = pos & 31; + } + + /* Connect id with pointer. */ + database->table[pos] = Pointer; + database->bitmap[n] |= (1u << i); + + *Id = pos + 1; + + database->nextId = (pos + 1) % database->capacity; + database->freeCount--; + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex)); + + gcmkFOOTER_ARG("*Id=%u", *Id); + return gcvSTATUS_OK; + +OnError: + if (table) + { + gckOS_Free(os, table); + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex)); + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_FreeIntegerId( + IN gctPOINTER Database, + IN gctUINT32 Id + ) +{ + gceSTATUS status; + gckINTEGERDB database = Database; + gckOS os = database->os; + gctUINT32 pos = Id - 1; + gctUINT32 n, i; + + gcmkHEADER_ARG("Database=%p Id=%d", Database, Id); + + gcmkVERIFY_OK(gckOS_AcquireMutex(os, database->mutex, gcvINFINITE)); + + n = pos >> 5; + i = pos & 31; + + if (pos >= database->capacity || (database->bitmap[n] & (1u << i)) == 0) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex)); + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + + /* Clear id. */ + database->bitmap[n] &= ~(1u << i); + database->table[pos] = gcvNULL; + + ++database->freeCount; + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_QueryIntegerId( + IN gctPOINTER Database, + IN gctUINT32 Id, + OUT gctPOINTER * Pointer + ) +{ + gceSTATUS status; + gckINTEGERDB database = Database; + gckOS os = database->os; + gctUINT32 pos = Id - 1; + gctUINT32 n, i; + + gcmkHEADER_ARG("Database=%p Id=%d", Database, Id); + gcmkVERIFY_ARGUMENT(Pointer != gcvNULL); + + gcmkVERIFY_OK(gckOS_AcquireMutex(os, database->mutex, gcvINFINITE)); + + n = pos >> 5; + i = pos & 31; + + if (pos >= database->capacity || (database->bitmap[n] & (1u << i)) == 0) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex)); + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + + *Pointer = database->table[pos]; + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex)); + + gcmkFOOTER_ARG("*Pointer=0x%08X", *Pointer); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + + +gctUINT32 +gckKERNEL_AllocateNameFromPointer( + IN gckKERNEL Kernel, + IN gctPOINTER Pointer + ) +{ + gceSTATUS status; + gctUINT32 name; + gctPOINTER database = Kernel->db->pointerDatabase; + + gcmkHEADER_ARG("Kernel=%p Pointer=%p", Kernel, Pointer); + + gcmkONERROR(gckKERNEL_AllocateIntegerId(database, Pointer, &name)); + + gcmkFOOTER_ARG("name=%d", name); + return name; + +OnError: + gcmkFOOTER(); + return 0; +} + +gctPOINTER +gckKERNEL_QueryPointerFromName( + IN gckKERNEL Kernel, + IN gctUINT32 Name + ) +{ + gceSTATUS status; + gctPOINTER pointer = gcvNULL; + gctPOINTER database = Kernel->db->pointerDatabase; + + gcmkHEADER_ARG("Kernel=%p Name=%d", Kernel, Name); + + /* Lookup in database to get pointer. */ + gcmkONERROR(gckKERNEL_QueryIntegerId(database, Name, &pointer)); + + gcmkFOOTER_ARG("pointer=0x%X", pointer); + return pointer; + +OnError: + gcmkFOOTER(); + return gcvNULL; +} + +gceSTATUS +gckKERNEL_DeleteName( + IN gckKERNEL Kernel, + IN gctUINT32 Name + ) +{ + gctPOINTER database = Kernel->db->pointerDatabase; + + gcmkHEADER_ARG("Kernel=%p Name=%p", Kernel, Name); + + /* Free name if exists. */ + gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(database, Name)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +***** Shared Buffer ************************************************************ +*******************************************************************************/ + +/******************************************************************************* +** +** gckKERNEL_CreateShBuffer +** +** Create shared buffer. +** The shared buffer can be used across processes. Other process needs call +** gckKERNEL_MapShBuffer before use it. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctUINT32 Size +** Specify the shared buffer size. +** +** OUTPUT: +** +** gctSHBUF * ShBuf +** Pointer to hold return shared buffer handle. +*/ +gceSTATUS +gckKERNEL_CreateShBuffer( + IN gckKERNEL Kernel, + IN gctUINT32 Size, + OUT gctSHBUF * ShBuf + ) +{ + gceSTATUS status; + gcsSHBUF_PTR shBuf = gcvNULL; + + gcmkHEADER_ARG("Kernel=%p, Size=%u", Kernel, Size); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + + if (Size == 0) + { + /* Invalid size. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + else if (Size > 1024) + { + /* Limite shared buffer size. */ + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + + /* Create a shared buffer structure. */ + gcmkONERROR( + gckOS_Allocate(Kernel->os, + sizeof (gcsSHBUF), + (gctPOINTER *)&shBuf)); + + /* Initialize shared buffer. */ + shBuf->id = 0; + shBuf->reference = gcvNULL; + shBuf->size = Size; + shBuf->data = gcvNULL; + + /* Allocate integer id for this shared buffer. */ + gcmkONERROR( + gckKERNEL_AllocateIntegerId(Kernel->db->pointerDatabase, + shBuf, + &shBuf->id)); + + /* Allocate atom. */ + gcmkONERROR(gckOS_AtomConstruct(Kernel->os, &shBuf->reference)); + + /* Set default reference count to 1. */ + gcmkVERIFY_OK(gckOS_AtomSet(Kernel->os, shBuf->reference, 1)); + + /* Return integer id. */ + *ShBuf = (gctSHBUF)(gctUINTPTR_T)shBuf->id; + + gcmkFOOTER_ARG("*ShBuf=%u", shBuf->id); + return gcvSTATUS_OK; + +OnError: + /* Error roll back. */ + if (shBuf != gcvNULL) + { + if (shBuf->id != 0) + { + gcmkVERIFY_OK( + gckKERNEL_FreeIntegerId(Kernel->db->pointerDatabase, + shBuf->id)); + } + + gcmkOS_SAFE_FREE(Kernel->os, shBuf); + } + + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckKERNEL_DestroyShBuffer +** +** Destroy shared buffer. +** This will decrease reference of specified shared buffer and do actual +** destroy when no reference on it. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctSHBUF ShBuf +** Specify the shared buffer to be destroyed. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_DestroyShBuffer( + IN gckKERNEL Kernel, + IN gctSHBUF ShBuf + ) +{ + gceSTATUS status; + gcsSHBUF_PTR shBuf; + gctINT32 oldValue = 0; + + gcmkHEADER_ARG("Kernel=%p ShBuf=%u", + Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL); + + /* Find shared buffer structure. */ + gcmkONERROR( + gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase, + (gctUINT32)(gctUINTPTR_T)ShBuf, + (gctPOINTER)&shBuf)); + + gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf); + + /* Decrease the reference count. */ + gckOS_AtomDecrement(Kernel->os, shBuf->reference, &oldValue); + + if (oldValue == 1) + { + /* Free integer id. */ + gcmkVERIFY_OK( + gckKERNEL_FreeIntegerId(Kernel->db->pointerDatabase, + shBuf->id)); + + /* Free atom. */ + gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, shBuf->reference)); + + if (shBuf->data) + { + gcmkOS_SAFE_FREE(Kernel->os, shBuf->data); + shBuf->data = gcvNULL; + } + + /* Free the shared buffer. */ + gcmkOS_SAFE_FREE(Kernel->os, shBuf); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckKERNEL_MapShBuffer +** +** Map shared buffer into this process so that it can be used in this process. +** This will increase reference count on the specified shared buffer. +** Call gckKERNEL_DestroyShBuffer to dereference. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctSHBUF ShBuf +** Specify the shared buffer to be mapped. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_MapShBuffer( + IN gckKERNEL Kernel, + IN gctSHBUF ShBuf + ) +{ + gceSTATUS status; + gcsSHBUF_PTR shBuf; + gctINT32 oldValue = 0; + + gcmkHEADER_ARG("Kernel=%p ShBuf=%u", + Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL); + + /* Find shared buffer structure. */ + gcmkONERROR( + gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase, + (gctUINT32)(gctUINTPTR_T)ShBuf, + (gctPOINTER)&shBuf)); + + gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf); + + /* Increase the reference count. */ + gckOS_AtomIncrement(Kernel->os, shBuf->reference, &oldValue); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckKERNEL_WriteShBuffer +** +** Write user data into shared buffer. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctSHBUF ShBuf +** Specify the shared buffer to be written to. +** +** gctPOINTER UserData +** User mode pointer to hold the source data. +** +** gctUINT32 ByteCount +** Specify number of bytes to write. If this is larger than +** shared buffer size, gcvSTATUS_INVALID_ARGUMENT is returned. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_WriteShBuffer( + IN gckKERNEL Kernel, + IN gctSHBUF ShBuf, + IN gctPOINTER UserData, + IN gctUINT32 ByteCount + ) +{ + gceSTATUS status; + gcsSHBUF_PTR shBuf = gcvNULL; + + gcmkHEADER_ARG("Kernel=%p ShBuf=%u UserData=%p ByteCount=%u", + Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf, UserData, ByteCount); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL); + + /* Find shared buffer structure. */ + gcmkONERROR( + gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase, + (gctUINT32)(gctUINTPTR_T)ShBuf, + (gctPOINTER)&shBuf)); + + gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf); + + if ((ByteCount > shBuf->size) || + (ByteCount == 0) || + (UserData == gcvNULL)) + { + /* Exceeds buffer max size or invalid. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + if (shBuf->data == gcvNULL) + { + /* Allocate buffer data when first time write. */ + gcmkONERROR(gckOS_Allocate(Kernel->os, ByteCount, &shBuf->data)); + } + + /* Copy data from user. */ + gcmkONERROR( + gckOS_CopyFromUserData(Kernel->os, + shBuf->data, + UserData, + ByteCount)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (shBuf && shBuf->data) + { + gcmkOS_SAFE_FREE(Kernel->os, shBuf->data); + shBuf->data = gcvNULL; + } + + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckKERNEL_ReadShBuffer +** +** Read data from shared buffer and copy to user pointer. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctSHBUF ShBuf +** Specify the shared buffer to be read from. +** +** gctPOINTER UserData +** User mode pointer to save output data. +** +** gctUINT32 ByteCount +** Specify number of bytes to read. +** If this is larger than shared buffer size, only avaiable bytes are +** copied. If smaller, copy requested size. +** +** OUTPUT: +** +** gctUINT32 * BytesRead +** Pointer to hold how many bytes actually read from shared buffer. +*/ +gceSTATUS +gckKERNEL_ReadShBuffer( + IN gckKERNEL Kernel, + IN gctSHBUF ShBuf, + IN gctPOINTER UserData, + IN gctUINT32 ByteCount, + OUT gctUINT32 * BytesRead + ) +{ + gceSTATUS status; + gcsSHBUF_PTR shBuf; + gctUINT32 bytes; + + gcmkHEADER_ARG("Kernel=%p ShBuf=%u UserData=%p ByteCount=%u", + Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf, UserData, ByteCount); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL); + + /* Find shared buffer structure. */ + gcmkONERROR( + gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase, + (gctUINT32)(gctUINTPTR_T)ShBuf, + (gctPOINTER)&shBuf)); + + gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf); + + if (shBuf->data == gcvNULL) + { + *BytesRead = 0; + + /* No data in shared buffer, skip copy. */ + status = gcvSTATUS_SKIP; + goto OnError; + } + else if (ByteCount == 0) + { + /* Invalid size to read. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + /* Determine bytes to copy. */ + bytes = (ByteCount < shBuf->size) ? ByteCount : shBuf->size; + + /* Copy data to user. */ + gcmkONERROR( + gckOS_CopyToUserData(Kernel->os, + shBuf->data, + UserData, + bytes)); + + /* Return copied size. */ + *BytesRead = bytes; + + gcmkFOOTER_ARG("*BytesRead=%u", bytes); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +/*******************************************************************************\ +*************************** List Helper ***************************************** +\*******************************************************************************/ + +static void +_ListAdd( + gcsLISTHEAD_PTR New, + gcsLISTHEAD_PTR Prev, + gcsLISTHEAD_PTR Next + ) +{ + Next->prev = New; + New->next = Next; + New->prev = Prev; + Prev->next = New; +} + +void +_ListDel( + gcsLISTHEAD_PTR Prev, + gcsLISTHEAD_PTR Next + ) +{ + Next->prev = Prev; + Prev->next = Next; +} + +void +gcsLIST_Init( + gcsLISTHEAD_PTR Node + ) +{ + Node->prev = Node; + Node->next = Node; +} + +void +gcsLIST_Add( + gcsLISTHEAD_PTR New, + gcsLISTHEAD_PTR Head + ) +{ + _ListAdd(New, Head, Head->next); +} + +void +gcsLIST_AddTail( + gcsLISTHEAD_PTR New, + gcsLISTHEAD_PTR Head + ) +{ + _ListAdd(New, Head->prev, Head); +} + +void +gcsLIST_Del( + gcsLISTHEAD_PTR Node + ) +{ + _ListDel(Node->prev, Node->next); +} + +gctBOOL +gcsLIST_Empty( + gcsLISTHEAD_PTR Head + ) +{ + return Head->next == Head; +} + +/*******************************************************************************\ +********************************* Fence ***************************************** +\*******************************************************************************/ + +gceSTATUS +gckFENCE_Create( + IN gckOS Os, + IN gckKERNEL Kernel, + OUT gckFENCE * Fence + ) +{ + gceSTATUS status; + +#if !gcdCAPTURE_ONLY_MODE + gcePOOL pool = gcvPOOL_DEFAULT; +#else + gcePOOL pool = gcvPOOL_VIRTUAL; +#endif + + gckFENCE fence = gcvNULL; + gctSIZE_T size = 8; + gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag |= gcvALLOC_FLAG_CACHEABLE; +#endif + + + gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcsFENCE), (gctPOINTER *)&fence)); + gcmkONERROR(gckOS_ZeroMemory(fence, gcmSIZEOF(gcsFENCE))); + gcmkONERROR(gckOS_CreateMutex(Os, (gctPOINTER *)&fence->mutex)); + + fence->kernel = Kernel; + + /* Allocate video memory node for fence. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + Kernel, + 64, + gcvVIDMEM_TYPE_FENCE, + allocFlag, + &size, + &pool, + &fence->videoMem + )); + + /* Lock for GPU access. */ + gcmkONERROR(gckVIDMEM_NODE_Lock( + Kernel, + fence->videoMem, + &fence->address + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + Kernel, + fence->videoMem, + gcvFALSE, + gcvFALSE, + &fence->logical + )); + + gcsLIST_Init(&fence->waitingList); + + *Fence = fence; + + return gcvSTATUS_OK; +OnError: + if (fence) + { + gckFENCE_Destory(Os, fence); + } + + return status; +} + +gceSTATUS +gckFENCE_Destory( + IN gckOS Os, + OUT gckFENCE Fence + ) +{ + if (Fence->mutex) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(Os, Fence->mutex)); + } + + if (Fence->logical) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + Fence->kernel, + Fence->videoMem, + 0, + gcvFALSE, + gcvFALSE + )); + + /* Synchronueous unlock. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( + Fence->kernel, + Fence->videoMem, + 0, + gcvNULL + )); + + /* Free video memory. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + Fence->kernel, + Fence->videoMem + )); + } + + gcmkOS_SAFE_FREE(Os, Fence); + + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckFENCE_Signal +** +** Signal all completed nodes. +** +** +*/ +gceSTATUS +gckFENCE_Signal( + IN gckOS Os, + IN gckFENCE Fence + ) +{ + gcsLISTHEAD_PTR list = &Fence->waitingList; + gcsLISTHEAD_PTR nodeHead, nodeTemp; + gckFENCE_SYNC sync; + gckOS os = Os; + gctUINT64 stamp = *(gctUINT64 *)Fence->logical; + + gcmkVERIFY_OK(gckOS_AcquireMutex(os, Fence->mutex, gcvINFINITE)); + + gcmkLIST_FOR_EACH_SAFE(nodeHead, nodeTemp, list) + { + sync = gcmCONTAINEROF(nodeHead, struct _gcsFENCE_SYNC, head); + + /* Signal all nodes which are complete. */ + if (sync->commitStamp <= stamp && sync->inList) + { + /* Signal. */ + gckOS_Signal(os, sync->signal, gcvTRUE); + + /* Remove from wait list. */ + gcsLIST_Del(nodeHead); + + /* Mark node not in waiting list. */ + sync->inList = gcvFALSE; + } + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Fence->mutex)); + + return gcvSTATUS_OK; +} + +gceSTATUS +gckDEVICE_Construct( + IN gckOS Os, + OUT gckDEVICE * Device + ) +{ + gceSTATUS status; + gckDEVICE device; + gctUINT i, j; + + gcmkHEADER(); + + gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcsDEVICE), (gctPOINTER *)&device)); + + gckOS_ZeroMemory(device, gcmSIZEOF(gcsDEVICE)); + + for (i = 0; i < gcvCORE_COUNT; i++) + { + device->coreInfoArray[i].type = gcvHARDWARE_INVALID; + + /* Initialize internal SRAM. */ + for (j = 0; j < gcvSRAM_INTER_COUNT; j++) + { + device->sRAMBases[i][j] = gcvINVALID_PHYSICAL_ADDRESS; + device->sRAMSizes[i][j] = 0; + device->sRAMPhysFaked[i][j] = gcvFALSE; + } + } + + /* Initialize external SRAM. */ + for (i = 0; i < gcvSRAM_EXT_COUNT; i++) + { + device->extSRAMGPUBases[i] = gcvINVALID_PHYSICAL_ADDRESS; + device->extSRAMBases[i] = gcvINVALID_PHYSICAL_ADDRESS; + device->extSRAMSizes[i] = 0; + } + + device->defaultHwType = gcvHARDWARE_INVALID; + + gcmkONERROR(gckOS_CreateMutex(Os, &device->stuckDumpMutex)); + gcmkONERROR(gckOS_CreateMutex(Os, &device->commitMutex)); + + device->os = Os; + device->showSRAMMapInfo = 0; + + *Device = device; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + + if (device != gcvNULL) + { + gckDEVICE_Destroy(Os, device); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckDEVICE_AddCore( + IN gckDEVICE Device, + IN gceCORE Core, + IN gctUINT ChipID, + IN gctPOINTER Context, + IN gckKERNEL * Kernel + ) +{ + gceSTATUS status; + gcsCORE_INFO * info = Device->coreInfoArray; + gceHARDWARE_TYPE type = (gceHARDWARE_TYPE)((gctUINT)gcvHARDWARE_INVALID); + gctUINT32 index = Device->coreNum; + gctUINT32 i; + gcsCORE_LIST *coreList; + gceHARDWARE_TYPE kernelType; + gceHARDWARE_TYPE defaultHwType; + gckKERNEL kernel; + + gcmkHEADER_ARG("Device=%p Core=%d ChipID=%d Context=%p Kernel=%p", + Device, Core, ChipID, Context, Kernel); + + gcmkASSERT(Device->coreNum < gcvCORE_COUNT); + + if (Core >= gcvCORE_MAJOR && Core <= gcvCORE_3D_MAX) + { + /* Chip ID is only used for 3D cores. */ + if (ChipID == gcvCHIP_ID_DEFAULT) + { + /* Apply default chipID if it is not set. */ + ChipID = Core; + } + } + + /* Construct gckKERNEL for this core. */ + gcmkONERROR(gckKERNEL_Construct( + Device->os, Core, ChipID, Context, Device, Device->database, Kernel)); + + kernel = *Kernel; + + if (Device->database == gcvNULL) + { + Device->database = kernel->db; + } + + gcmkVERIFY_OK( + gckKERNEL_GetHardwareType(kernel, + &kernelType)); + + if (kernelType >= gcvHARDWARE_NUM_TYPES) + { + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + info[index].type = kernelType; + info[index].core = Core; + info[index].kernel = kernel; + info[index].chipID = ChipID; + + if (index == 0) + { + /* First core, map all type/core to it. */ + for (; type != gcvHARDWARE_NUM_TYPES; type = (gceHARDWARE_TYPE)((gctUINT)type + 1)) + { + Device->map[type].num = 0; + + for (i = 0 ; i < 4; i++) + { + Device->map[type].kernels[i] = kernel; + } + } + } + + /* Get core list of this type. */ + coreList = &Device->map[kernelType]; + + /* Setup gceHARDWARE_TYPE to gceCORE mapping. */ + coreList->kernels[coreList->num++] = kernel; + + defaultHwType = kernelType; + if (kernelType == gcvHARDWARE_3D2D) + { + coreList = &Device->map[gcvHARDWARE_3D]; + coreList->kernels[coreList->num++] = kernel; + defaultHwType = gcvHARDWARE_3D; + } + + /* Advance total core number. */ + Device->coreNum++; + + /* Default HW type was chosen: 3D > 2D > VG */ + if (Device->defaultHwType == gcvHARDWARE_INVALID) + { + Device->defaultHwType = defaultHwType; + } + else if (Device->defaultHwType > defaultHwType) + { + Device->defaultHwType = defaultHwType; + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckDEVICE_ChipInfo( + IN gckDEVICE Device, + IN gcsHAL_INTERFACE_PTR Interface + ) +{ + { + gctUINT i; + gcsCORE_INFO * info = Device->coreInfoArray; + + for (i = 0; i < Device->coreNum; i++) + { + Interface->u.ChipInfo.types[i] = info[i].type; + Interface->u.ChipInfo.ids[i] = info[i].chipID; + + Interface->u.ChipInfo.coreIndexs[i] = info[i].core; + } + + Interface->u.ChipInfo.count = Device->coreNum; + } + + return gcvSTATUS_OK; +} + +gceSTATUS +gckDEVICE_Version( + IN gckDEVICE Device, + IN gcsHAL_INTERFACE_PTR Interface + ) +{ + Interface->u.Version.major = gcvVERSION_MAJOR; + Interface->u.Version.minor = gcvVERSION_MINOR; + Interface->u.Version.patch = gcvVERSION_PATCH; + Interface->u.Version.build = gcvVERSION_BUILD; +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL, + "KERNEL version %s", gcvVERSION_STRING); +#endif + + return gcvSTATUS_OK; +} + +gceSTATUS +gckDEVICE_Destroy( + IN gckOS Os, + IN gckDEVICE Device + ) +{ + gctINT i; + gcsCORE_INFO * info = Device->coreInfoArray; + + for (i = Device->coreNum - 1; i >= 0 ; i--) + { + if (info[i].kernel != gcvNULL) + { + gckKERNEL_Destroy(info[i].kernel); + } + } + + if (Device->commitMutex) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(Os, Device->commitMutex)); + } + if (Device->stuckDumpMutex) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(Os, Device->stuckDumpMutex)); + } + + gcmkOS_SAFE_FREE(Os, Device); + + return gcvSTATUS_OK; +} + +static gceSTATUS +gckDEVICE_SetTimeOut( + IN gckDEVICE Device, + IN gcsHAL_INTERFACE_PTR Interface + ) +{ +#if gcdGPU_TIMEOUT + gckKERNEL kernel; + gctUINT i; + gceHARDWARE_TYPE type = Interface->hardwareType; + gcsCORE_LIST *coreList; + gctUINT32 processID = 0; + gcsCORE_INFO *info = Device->coreInfoArray; + + coreList = &Device->map[type]; + + /* Get the current process ID. */ + gckOS_GetProcessID(&processID); + + for (i = 0; i < Device->coreNum; i++) + { + if (type == gcvHARDWARE_3D || type == gcvHARDWARE_3D2D || type == gcvHARDWARE_VIP) + { + kernel = info[i].kernel; + } + else + { + kernel = coreList->kernels[i]; + } + + kernel->timeOut = Interface->u.SetTimeOut.timeOut; + + kernel->timeoutPID = processID; + } +#endif + + return gcvSTATUS_OK; +} + + +gceSTATUS +gckDEVICE_Dispatch( + IN gckDEVICE Device, + IN gcsHAL_INTERFACE_PTR Interface + ) +{ + gceSTATUS status = gcvSTATUS_NOT_SUPPORTED; + gckKERNEL kernel; + gceHARDWARE_TYPE type = Interface->hardwareType; + gctUINT32 coreIndex = Interface->coreIndex; + + switch (Interface->command) + { + case gcvHAL_CHIP_INFO: + status = gckDEVICE_ChipInfo(Device, Interface); + break; + + case gcvHAL_VERSION: + status = gckDEVICE_Version(Device, Interface); + break; + + case gcvHAL_SET_TIMEOUT: + status = gckDEVICE_SetTimeOut(Device, Interface); + break; + + default: + status = gcvSTATUS_NOT_SUPPORTED; + break; + } + + if (gcmIS_SUCCESS(status)) + { + /* Dispatch handled in this layer. */ + Interface->status = status; + } + else + { + /* Need go through gckKERNEL dispatch. */ + if (type == gcvHARDWARE_3D || type == gcvHARDWARE_3D2D || type == gcvHARDWARE_VIP) + { + kernel = Device->coreInfoArray[coreIndex].kernel; + } + else + { + kernel = Device->map[type].kernels[coreIndex]; + } + + { + status = gckKERNEL_Dispatch(kernel, Device, Interface); + } + + /* Interface->status is handled in gckKERNEL_Dispatch(). */ + } + + return status; +} + +gceSTATUS +gckDEVICE_GetMMU( + IN gckDEVICE Device, + IN gceHARDWARE_TYPE Type, + IN gckMMU *Mmu + ) +{ + gcmkHEADER(); + gcmkVERIFY_ARGUMENT(Type < gcvHARDWARE_NUM_TYPES); + + *Mmu = Device->mmus[Type]; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckDEVICE_SetMMU( + IN gckDEVICE Device, + IN gceHARDWARE_TYPE Type, + IN gckMMU Mmu + ) +{ + gcmkHEADER(); + gcmkVERIFY_ARGUMENT(Type < gcvHARDWARE_NUM_TYPES); + + Device->mmus[Type] = Mmu; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +#if gcdENABLE_TRUST_APPLICATION +gceSTATUS +gckKERNEL_MapInTrustApplicaiton( + IN gckKERNEL Kernel, + IN gctPOINTER Logical, + IN gctPHYS_ADDR Physical, + IN gctUINT32 GPUAddress, + IN gctSIZE_T PageCount + ) +{ + gceSTATUS status; + gctUINT32 * physicalArrayLogical = gcvNULL; + gctSIZE_T bytes; + gctPOINTER logical = Logical; + gctUINT32 i; + gctSIZE_T pageSize; + gctUINT32 pageMask; + + gcmkHEADER(); + + gcmkVERIFY_OK(gckOS_GetPageSize(Kernel->os, &pageSize)); + + pageMask = (gctUINT32)pageSize - 1; + + bytes = PageCount * gcmSIZEOF(gctUINT32); + + gcmkONERROR(gckOS_Allocate( + Kernel->os, + bytes, + (gctPOINTER *)&physicalArrayLogical + )); + + /* Fill in physical array. */ + for (i = 0; i < PageCount; i++) + { + gctPHYS_ADDR_T phys; + status = gckOS_GetPhysicalFromHandle( + Kernel->os, + Physical, + i * 4096, + &phys + ); + + if (status == gcvSTATUS_NOT_SUPPORTED) + { + gcmkONERROR(gckOS_GetPhysicalAddress( + Kernel->os, + logical, + &phys + )); + + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Kernel->os, phys, &phys)); + } + + phys &= ~pageMask; + + gcmkSAFECASTPHYSADDRT(physicalArrayLogical[i], phys); + + logical = (gctUINT8_PTR)logical + 4096; + } + + gcmkONERROR(gckKERNEL_SecurityMapMemory( + Kernel, + physicalArrayLogical, + 0, + (gctUINT32)PageCount, + &GPUAddress + )); + + gcmkVERIFY_OK(gckOS_Free( + Kernel->os, + physicalArrayLogical + )); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (physicalArrayLogical != gcvNULL) + { + gcmkVERIFY_OK(gckOS_Free( + Kernel->os, + (gctPOINTER)physicalArrayLogical + )); + } + + gcmkFOOTER(); + return status; +} +#endif +
diff --git a/hal/kernel/gc_hal_kernel.h b/hal/kernel/gc_hal_kernel.h new file mode 100644 index 0000000..7f25279 --- /dev/null +++ b/hal/kernel/gc_hal_kernel.h
@@ -0,0 +1,2419 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_kernel_h_ +#define __gc_hal_kernel_h_ + +#include "gc_hal.h" +#include "gc_hal_kernel_hardware.h" +#include "gc_hal_kernel_hardware_fe.h" +#include "shared/gc_hal_driver.h" +#include "gc_hal_kernel_mutex.h" +#include "gc_hal_metadata.h" + + +#if gcdSECURITY || gcdENABLE_TRUST_APPLICATION +#include "gc_hal_security_interface.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/******************************************************************************* +***** New MMU Defination *******************************************************/ + +#if gcdENABLE_MMU_1KMODE +/* 1k mode */ +#define gcdMMU_MTLB_SHIFT 24 +#define gcdMMU_STLB_4K_SHIFT 12 +#define gcdMMU_STLB_64K_SHIFT 16 +#define gcdMMU_STLB_1M_SHIFT 20 +#define gcdMMU_STLB_16M_SHIFT 24 + +#define gcdMMU_MTLB_BITS (32 - gcdMMU_MTLB_SHIFT) +#define gcdMMU_PAGE_4K_BITS gcdMMU_STLB_4K_SHIFT +#define gcdMMU_STLB_4K_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_PAGE_4K_BITS) +#define gcdMMU_PAGE_64K_BITS gcdMMU_STLB_64K_SHIFT +#define gcdMMU_STLB_64K_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_PAGE_64K_BITS) +#define gcdMMU_PAGE_1M_BITS gcdMMU_STLB_1M_SHIFT +#define gcdMMU_STLB_1M_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_PAGE_1M_BITS) +#define gcdMMU_PAGE_16M_BITS gcdMMU_STLB_16M_SHIFT +#define gcdMMU_STLB_16M_BITS 4 + + +#define gcdMMU_MTLB_ENTRY_NUM (1 << gcdMMU_MTLB_BITS) +#define gcdMMU_MTLB_SIZE (gcdMMU_MTLB_ENTRY_NUM << 2) +#define gcdMMU_STLB_4K_ENTRY_NUM (1 << gcdMMU_STLB_4K_BITS) +#define gcdMMU_STLB_4K_SIZE (gcdMMU_STLB_4K_ENTRY_NUM << 2) +#define gcdMMU_PAGE_4K_SIZE (1 << gcdMMU_STLB_4K_SHIFT) +#define gcdMMU_STLB_64K_ENTRY_NUM (1 << gcdMMU_STLB_64K_BITS) +#define gcdMMU_STLB_64K_SIZE (gcdMMU_STLB_64K_ENTRY_NUM << 2) +#define gcdMMU_PAGE_64K_SIZE (1 << gcdMMU_STLB_64K_SHIFT) +#define gcdMMU_STLB_1M_ENTRY_NUM (1 << gcdMMU_STLB_1M_BITS) +#define gcdMMU_STLB_1M_SIZE (gcdMMU_STLB_1M_ENTRY_NUM << 2) +#define gcdMMU_PAGE_1M_SIZE (1 << gcdMMU_STLB_1M_SHIFT) +#define gcdMMU_STLB_16M_ENTRY_NUM (1 << gcdMMU_STLB_16M_BITS) +#define gcdMMU_STLB_16M_SIZE (gcdMMU_STLB_16M_ENTRY_NUM << 2) +#define gcdMMU_PAGE_16M_SIZE (1 << gcdMMU_STLB_16M_SHIFT) + + +#define gcdMMU_MTLB_MASK (~((1U << gcdMMU_MTLB_SHIFT)-1)) +#define gcdMMU_STLB_4K_MASK ((~0U << gcdMMU_STLB_4K_SHIFT) ^ gcdMMU_MTLB_MASK) +#define gcdMMU_PAGE_4K_MASK (gcdMMU_PAGE_4K_SIZE - 1) +#define gcdMMU_STLB_64K_MASK ((~((1U << gcdMMU_STLB_64K_SHIFT)-1)) ^ gcdMMU_MTLB_MASK) +#define gcdMMU_PAGE_64K_MASK (gcdMMU_PAGE_64K_SIZE - 1) +#define gcdMMU_STLB_1M_MASK ((~((1U << gcdMMU_STLB_1M_SHIFT)-1)) ^ gcdMMU_MTLB_MASK) +#define gcdMMU_PAGE_1M_MASK (gcdMMU_PAGE_1M_SIZE - 1) +#define gcdMMU_STLB_16M_MASK 0x0F000000 +#define gcdMMU_PAGE_16M_MASK (gcdMMU_PAGE_16M_SIZE - 1) + + +/* Page offset definitions. */ +#define gcdMMU_OFFSET_4K_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_STLB_4K_BITS) +#define gcdMMU_OFFSET_4K_MASK ((1U << gcdMMU_OFFSET_4K_BITS) - 1) +#define gcdMMU_OFFSET_64K_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_STLB_64K_BITS) +#define gcdMMU_OFFSET_64K_MASK ((1U << gcdMMU_OFFSET_64K_BITS) - 1) +#define gcdMMU_OFFSET_1M_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_STLB_1M_BITS) +#define gcdMMU_OFFSET_1M_MASK ((1U << gcdMMU_OFFSET_1M_BITS) - 1) +#define gcdMMU_OFFSET_16M_BITS (32 - gcdMMU_MTLB_BITS) +#define gcdMMU_OFFSET_16M_MASK ((1U << gcdMMU_OFFSET_16M_BITS) - 1) + + +#define gcdMMU_MTLB_ENTRY_HINTS_BITS 6 +#define gcdMMU_MTLB_ENTRY_STLB_MASK (~((1U << gcdMMU_MTLB_ENTRY_HINTS_BITS) - 1)) + +#define gcdMMU_MTLB_PRESENT 0x00000001 +#define gcdMMU_MTLB_EXCEPTION 0x00000002 +#define gcdMMU_MTLB_4K_PAGE (0 << 2) +#define gcdMMU_MTLB_64K_PAGE (1 << 2) +#define gcdMMU_MTLB_1M_PAGE (2 << 2) +#define gcdMMU_MTBL_16M_PAGE (3 << 2) + +#define gcdMMU_STLB_PRESENT 0x00000001 +#define gcdMMU_STLB_EXCEPTION 0x00000002 +#define gcdMMU_STBL_WRITEABLE 0x00000004 + +#else /* 4K mode */ + +#define gcdMMU_MTLB_SHIFT 22 +#define gcdMMU_STLB_4K_SHIFT 12 +#define gcdMMU_STLB_64K_SHIFT 16 + +#define gcdMMU_MTLB_BITS (32 - gcdMMU_MTLB_SHIFT) +#define gcdMMU_PAGE_4K_BITS gcdMMU_STLB_4K_SHIFT +#define gcdMMU_STLB_4K_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_PAGE_4K_BITS) +#define gcdMMU_PAGE_64K_BITS gcdMMU_STLB_64K_SHIFT +#define gcdMMU_STLB_64K_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_PAGE_64K_BITS) + +#define gcdMMU_MTLB_ENTRY_NUM (1 << gcdMMU_MTLB_BITS) +#define gcdMMU_MTLB_SIZE (gcdMMU_MTLB_ENTRY_NUM << 2) +#define gcdMMU_STLB_4K_ENTRY_NUM (1 << gcdMMU_STLB_4K_BITS) +#define gcdMMU_STLB_4K_SIZE (gcdMMU_STLB_4K_ENTRY_NUM << 2) +#define gcdMMU_PAGE_4K_SIZE (1 << gcdMMU_STLB_4K_SHIFT) +#define gcdMMU_STLB_64K_ENTRY_NUM (1 << gcdMMU_STLB_64K_BITS) +#define gcdMMU_STLB_64K_SIZE (gcdMMU_STLB_64K_ENTRY_NUM << 2) +#define gcdMMU_PAGE_64K_SIZE (1 << gcdMMU_STLB_64K_SHIFT) + +#define gcdMMU_MTLB_MASK (~((1U << gcdMMU_MTLB_SHIFT)-1)) +#define gcdMMU_STLB_4K_MASK ((~0U << gcdMMU_STLB_4K_SHIFT) ^ gcdMMU_MTLB_MASK) +#define gcdMMU_PAGE_4K_MASK (gcdMMU_PAGE_4K_SIZE - 1) +#define gcdMMU_STLB_64K_MASK ((~((1U << gcdMMU_STLB_64K_SHIFT)-1)) ^ gcdMMU_MTLB_MASK) +#define gcdMMU_PAGE_64K_MASK (gcdMMU_PAGE_64K_SIZE - 1) + +/* Page offset definitions. */ +#define gcdMMU_OFFSET_4K_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_STLB_4K_BITS) +#define gcdMMU_OFFSET_4K_MASK ((1U << gcdMMU_OFFSET_4K_BITS) - 1) +#define gcdMMU_OFFSET_64K_BITS (32 - gcdMMU_MTLB_BITS - gcdMMU_STLB_64K_BITS) +#define gcdMMU_OFFSET_64K_MASK ((1U << gcdMMU_OFFSET_64K_BITS) - 1) + +#define gcdMMU_MTLB_ENTRY_HINTS_BITS 6 +#define gcdMMU_MTLB_ENTRY_STLB_MASK (~((1U << gcdMMU_MTLB_ENTRY_HINTS_BITS) - 1)) + +#define gcdMMU_MTLB_PRESENT 0x00000001 +#define gcdMMU_MTLB_EXCEPTION 0x00000002 +#define gcdMMU_MTLB_4K_PAGE (0 << 2) +#define gcdMMU_MTLB_64K_PAGE (1 << 2) + + +#define gcdMMU_STLB_PRESENT 0x00000001 +#define gcdMMU_STLB_EXCEPTION 0x00000002 +#define gcdMMU_STLB_WRITEABLE 0x00000004 + +#endif + +#define gcd1M_PAGE_SIZE (1 << 20) +#define gcd1M_PAGE_SHIFT 20 + +/******************************************************************************* +***** Stuck Dump Level ********************************************************/ + +/* Dump nonthing when stuck happens. */ +#define gcvSTUCK_DUMP_NONE 0 + +/* Dump GPU state and memory near stuck point. */ +#define gcvSTUCK_DUMP_NEARBY_MEMORY 1 + +/* Beside gcvSTUCK_DUMP_NEARBY_MEMORY, dump context buffer and user command buffer. */ +#define gcvSTUCK_DUMP_USER_COMMAND 2 + +/* Beside gcvSTUCK_DUMP_USER_COMMAND, commit will be stall +** to make sure command causing stuck isn't missed. */ +#define gcvSTUCK_DUMP_STALL_COMMAND 3 + +/* Beside gcvSTUCK_DUMP_USER_COMMAND, dump kernel command buffer. */ +#define gcvSTUCK_DUMP_ALL_COMMAND 4 + +/******************************************************************************* +***** Page table **************************************************************/ + +#define gcvPAGE_TABLE_DIRTY_BIT_OTHER (1 << 0) +#define gcvPAGE_TABLE_DIRTY_BIT_FE (1 << 1) + +/******************************************************************************* +***** Process Database Management *********************************************/ + +typedef enum _gceDATABASE_TYPE +{ + gcvDB_VIDEO_MEMORY = 1, /* Video memory created. */ + gcvDB_COMMAND_BUFFER, /* Command Buffer. */ + gcvDB_NON_PAGED, /* Non paged memory. */ + gcvDB_CONTIGUOUS, /* Contiguous memory. */ + gcvDB_SIGNAL, /* Signal. */ + gcvDB_VIDEO_MEMORY_LOCKED, /* Video memory locked. */ + gcvDB_CONTEXT, /* Context */ + gcvDB_IDLE, /* GPU idle. */ + gcvDB_MAP_MEMORY, /* Map memory */ + gcvDB_MAP_USER_MEMORY, /* Map user memory */ + gcvDB_SHBUF, /* Shared buffer. */ + + gcvDB_NUM_TYPES, +} +gceDATABASE_TYPE; + +#define gcdDATABASE_TYPE_MASK 0x000000FF +#define gcdDB_VIDEO_MEMORY_TYPE_MASK 0x0000FF00 +#define gcdDB_VIDEO_MEMORY_TYPE_SHIFT 8 + +#define gcdDB_VIDEO_MEMORY_POOL_MASK 0x00FF0000 +#define gcdDB_VIDEO_MEMORY_POOL_SHIFT 16 + +typedef struct _gcsDATABASE_RECORD * gcsDATABASE_RECORD_PTR; +typedef struct _gcsDATABASE_RECORD +{ + /* Pointer to kernel. */ + gckKERNEL kernel; + + /* Pointer to next database record. */ + gcsDATABASE_RECORD_PTR next; + + /* Type of record. */ + gceDATABASE_TYPE type; + + /* Data for record. */ + gctPOINTER data; + gctPHYS_ADDR physical; + gctSIZE_T bytes; +} +gcsDATABASE_RECORD; + +typedef struct _gcsDATABASE * gcsDATABASE_PTR; +typedef struct _gcsDATABASE +{ + /* Pointer to next entry is hash list. */ + gcsDATABASE_PTR next; + gctSIZE_T slot; + + /* Process ID. */ + gctUINT32 processID; + + /* Open-Close ref count */ + gctPOINTER refs; + + /* Already mark for delete and cannot reenter */ + gctBOOL deleted; + + /* Sizes to query. */ + gcsDATABASE_COUNTERS vidMem; + gcsDATABASE_COUNTERS nonPaged; + gcsDATABASE_COUNTERS contiguous; + gcsDATABASE_COUNTERS mapUserMemory; + gcsDATABASE_COUNTERS mapMemory; + + gcsDATABASE_COUNTERS vidMemType[gcvVIDMEM_TYPE_COUNT]; + /* Counter for each video memory pool. */ + gcsDATABASE_COUNTERS vidMemPool[gcvPOOL_NUMBER_OF_POOLS]; + gctPOINTER counterMutex; + + /* Idle time management. */ + gctUINT64 lastIdle; + gctUINT64 idle; + + /* Pointer to database. */ + gcsDATABASE_RECORD_PTR list[48]; + + gctPOINTER handleDatabase; + gctPOINTER handleDatabaseMutex; +} +gcsDATABASE; + +typedef struct _gcsFDPRIVATE * gcsFDPRIVATE_PTR; +typedef struct _gcsFDPRIVATE +{ + gctINT (* release) (gcsFDPRIVATE_PTR Private); +} +gcsFDPRIVATE; + +typedef struct _gcsRECORDER * gckRECORDER; + + +/* Create a process database that will contain all its allocations. */ +gceSTATUS +gckKERNEL_CreateProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID + ); + +/* Add a record to the process database. */ +gceSTATUS +gckKERNEL_AddProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gceDATABASE_TYPE Type, + IN gctPOINTER Pointer, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Size + ); + +/* Remove a record to the process database. */ +gceSTATUS +gckKERNEL_RemoveProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gceDATABASE_TYPE Type, + IN gctPOINTER Pointer + ); + +/* Destroy the process database. */ +gceSTATUS +gckKERNEL_DestroyProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID + ); + +/* Find a record to the process database. */ +gceSTATUS +gckKERNEL_FindProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 ThreadID, + IN gceDATABASE_TYPE Type, + IN gctPOINTER Pointer, + OUT gcsDATABASE_RECORD_PTR Record + ); + +/* Query the process database. */ +gceSTATUS +gckKERNEL_QueryProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctBOOL LastProcessID, + IN gceDATABASE_TYPE Type, + OUT gcuDATABASE_INFO * Info + ); + +/* Dump the process database. */ +gceSTATUS +gckKERNEL_DumpProcessDB( + IN gckKERNEL Kernel + ); + +/* Dump the video memory usage for process specified. */ +gceSTATUS +gckKERNEL_DumpVidMemUsage( + IN gckKERNEL Kernel, + IN gctINT32 ProcessID + ); + +gceSTATUS +gckKERNEL_FindDatabase( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctBOOL LastProcessID, + OUT gcsDATABASE_PTR * Database + ); + +gceSTATUS +gckKERNEL_FindHandleDatbase( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + OUT gctPOINTER * HandleDatabase, + OUT gctPOINTER * HandleDatabaseMutex + ); + +gceSTATUS +gckMMU_GetPageEntry( + IN gckMMU Mmu, + IN gcePAGE_TYPE PageType, + IN gctUINT32 Address, + IN gctUINT32_PTR *PageTable + ); + +gceSTATUS +gckMMU_SetupSRAM( + IN gckMMU Mmu, + IN gckHARDWARE Hardware, + IN gckDEVICE Device + ); + +gceSTATUS +gckMMU_SetupDynamicSpace( + IN gckMMU Mmu + ); + +void +gckMMU_DumpRecentFreedAddress( + IN gckMMU Mmu + ); + +gceSTATUS +gckKERNEL_CreateIntegerDatabase( + IN gckKERNEL Kernel, + IN gctUINT32 Capacity, + OUT gctPOINTER * Database + ); + +gceSTATUS +gckKERNEL_DestroyIntegerDatabase( + IN gckKERNEL Kernel, + IN gctPOINTER Database + ); + +gceSTATUS +gckKERNEL_AllocateIntegerId( + IN gctPOINTER Database, + IN gctPOINTER Pointer, + OUT gctUINT32 * Id + ); + +gceSTATUS +gckKERNEL_FreeIntegerId( + IN gctPOINTER Database, + IN gctUINT32 Id + ); + +gceSTATUS +gckKERNEL_QueryIntegerId( + IN gctPOINTER Database, + IN gctUINT32 Id, + OUT gctPOINTER * Pointer + ); + +/* Pointer rename */ +gctUINT32 +gckKERNEL_AllocateNameFromPointer( + IN gckKERNEL Kernel, + IN gctPOINTER Pointer + ); + +gctPOINTER +gckKERNEL_QueryPointerFromName( + IN gckKERNEL Kernel, + IN gctUINT32 Name + ); + +gceSTATUS +gckKERNEL_DeleteName( + IN gckKERNEL Kernel, + IN gctUINT32 Name + ); + +/******************************************************************************* +********* Timer Management ****************************************************/ +typedef struct _gcsTIMER * gcsTIMER_PTR; +typedef struct _gcsTIMER +{ + /* Start and Stop time holders. */ + gctUINT64 startTime; + gctUINT64 stopTime; +} +gcsTIMER; + +/******************************************************************************\ +********************************** Structures ********************************** +\******************************************************************************/ + +/* gckDB object. */ +struct _gckDB +{ + /* Database management. */ + gcsDATABASE_PTR db[16]; + gctPOINTER dbMutex; + gcsDATABASE_PTR freeDatabase; + gcsDATABASE_RECORD_PTR freeRecord; + gcsDATABASE_PTR lastDatabase; + gctUINT32 lastProcessID; + gctUINT64 lastIdle; + gctUINT64 idleTime; + gctUINT64 lastSlowdown; + gctUINT64 lastSlowdownIdle; + + gctPOINTER nameDatabase; + gctPOINTER nameDatabaseMutex; + + gctPOINTER pointerDatabase; + + gcsLISTHEAD videoMemList; + gctPOINTER videoMemListMutex; +}; + +typedef struct _gckCOMMAND * gckCOMMAND; + +typedef struct _gckEVENT * gckEVENT; + +/* gckKERNEL object. */ +struct _gckKERNEL +{ + /* Object. */ + gcsOBJECT object; + + /* Pointer to gckOS object. */ + gckOS os; + + /* Pointer to gckDEVICE object. */ + gckDEVICE device; + + /* Pointer to gckHARDWARE object. */ + gckHARDWARE hardware; + + /* Core */ + gceCORE core; + gctUINT chipID; + + /* Main command module, event and context. */ + gckCOMMAND command; + gckEVENT eventObj; + gctPOINTER context; + + /* Async FE command and event modules. */ + gckCOMMAND asyncCommand; + gckEVENT asyncEvent; + + /* Pointer to gckMMU object. */ + gckMMU mmu; + + /* Arom holding number of clients. */ + gctPOINTER atomClients; + +#if VIVANTE_PROFILER + /* Enable profiling */ + gctBOOL profileEnable; + /* Clear profile register or not*/ + gctBOOL profileCleanRegister; +#endif + +#ifdef QNX_SINGLE_THREADED_DEBUGGING + gctPOINTER debugMutex; +#endif + + /* Database management. */ + gckDB db; + gctBOOL dbCreated; + + gctUINT64 resetTimeStamp; + + /* Pointer to gckEVENT object. */ + gcsTIMER timers[8]; + gctUINT32 timeOut; + + +#if gcdDVFS + gckDVFS dvfs; +#endif + +#if gcdLINUX_SYNC_FILE + gctHANDLE timeline; +#endif + + /* Enable recovery. */ + gctBOOL recovery; + + /* Level of dump information after stuck. */ + gctUINT stuckDump; + +#if gcdSECURITY || gcdENABLE_TRUST_APPLICATION + gctUINT32 securityChannel; +#endif + + /* Timer to monitor GPU stuck. */ + gctPOINTER monitorTimer; + + /* Flag to quit monitor timer. */ + gctBOOL monitorTimerStop; + + /* Monitor states. */ + gctBOOL monitoring; + gctUINT32 lastCommitStamp; + gctUINT32 timer; + gctUINT32 restoreAddress; + gctINT32 restoreMask; + + gckVIDMEM_BLOCK vidMemBlock; + gctPOINTER vidMemBlockMutex; + + gctUINT32 contiguousBaseAddress; + gctUINT32 externalBaseAddress; + gctUINT32 internalBaseAddress; + + /* External shared SRAM. */ + gctUINT32 extSRAMBaseAddresses[gcvSRAM_EXT_COUNT]; + gctUINT32 extSRAMIndex; + + /* Per core SRAM description. */ + gctUINT32 sRAMIndex; + gckVIDMEM sRAMVidMem[gcvSRAM_INTER_COUNT]; + gctPHYS_ADDR sRAMPhysical[gcvSRAM_INTER_COUNT]; + gctUINT32 sRAMBaseAddresses[gcvSRAM_INTER_COUNT]; + gctUINT32 sRAMSizes[gcvSRAM_INTER_COUNT]; + gctBOOL sRAMPhysFaked[gcvSRAM_INTER_COUNT]; + gctUINT64 sRAMLoopMode; + + gctUINT32 timeoutPID; + gctBOOL threadInitialized; +}; + +struct _FrequencyHistory +{ + gctUINT32 frequency; + gctUINT32 count; +}; + +/* gckDVFS object. */ +struct _gckDVFS +{ + gckOS os; + gckHARDWARE hardware; + gctPOINTER timer; + gctUINT32 pollingTime; + gctBOOL stop; + gctUINT32 totalConfig; + gctUINT32 loads[8]; + gctUINT8 currentScale; + struct _FrequencyHistory frequencyHistory[16]; +}; + +typedef struct _gcsFENCE * gckFENCE; +typedef struct _gcsFENCE +{ + /* Pointer to required object. */ + gckKERNEL kernel; + + /* Fence location. */ + gckVIDMEM_NODE videoMem; + gctPOINTER logical; + gctUINT32 address; + + gcsLISTHEAD waitingList; + gctPOINTER mutex; +} +gcsFENCE; + +/* A sync point attached to fence. */ +typedef struct _gcsFENCE_SYNC * gckFENCE_SYNC; +typedef struct _gcsFENCE_SYNC +{ + /* Stamp of commit access this node. */ + gctUINT64 commitStamp; + + /* Attach to waiting list. */ + gcsLISTHEAD head; + + gctPOINTER signal; + + gctBOOL inList; +} +gcsFENCE_SYNC; + +typedef struct _gcsCOMMAND_QUEUE +{ + gctSIGNAL signal; + gckVIDMEM_NODE videoMem; + gctPOINTER logical; + gctUINT32 address; +} +gcsCOMMAND_QUEUE; + +/* gckCOMMAND object. */ +struct _gckCOMMAND +{ + /* Object. */ + gcsOBJECT object; + + /* Pointer to required object. */ + gckKERNEL kernel; + gckOS os; + + gceHW_FE_TYPE feType; + + /* Number of bytes per page. */ + gctUINT32 pageSize; + + /* Current pipe select. */ + gcePIPE_SELECT pipeSelect; + + /* Command queue running flag. */ + gctBOOL running; + + /* Idle flag and commit stamp. */ + gctBOOL idle; + gctUINT64 commitStamp; + + /* Command queue mutex. */ + gctPOINTER mutexQueue; + + /* Context switching mutex. */ + gctPOINTER mutexContext; + + /* Context sequence mutex. */ + gctPOINTER mutexContextSeq; + + /* Command queue power semaphore. */ + gctPOINTER powerSemaphore; + + /* Command queues. */ + gcsCOMMAND_QUEUE queues[gcdCOMMAND_QUEUES]; + + /* Current queue. */ + gckVIDMEM_NODE videoMem; + gctPOINTER logical; + gctUINT32 address; + + gctUINT32 offset; + gctINT index; +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gctUINT wrapCount; +#endif + + /* The command queue is new. */ + gctBOOL newQueue; + + /* Context management. */ + gckCONTEXT currContext; + gctPOINTER stateMap; + + /* Pointer to last WAIT command. */ + /* Wait-Link FE only. */ + struct + { + gckVIDMEM_NODE videoMem; + gctUINT32 offset; + gctPOINTER logical; + gctUINT32 address; + gctUINT32 size; + } + waitPos; + + /* MCFE. */ + gctUINT32 totalSemaId; + + /* + * freeSemaId ------> [pendingSema] + * ^ freePos -->+ + + * | ^ | | + * | | | | + * | nextPos ---+ | + * | | + * | | + * | v + * nextSemaId <----------------------+ + * + * Terminology: + * 'freeSemaId': the top (final) free sema id can be used, signaled already. + * 'nextSemaId': the next sema id to be used. + * 'pendingSema': the used semaphores which are tracked in the ring. + * + * 3 Id sections: + * Id(s) between 'nextSemaId' and 'freeSemaId': + * Available for immediate use. + * + * Id(s) between 'freeSemaId' (exclusive) and 'pendingSema': + * To be signed, will be available to use later. + * + * Id(s) between 'pendingSema' and 'nextSemaId': + * The semaphores just used, not tracking in pendingSema ring. + * + * Conditions: + * 'nextSemaId' = 'freeSemaId': + * Using the final free semaphore, means the ring is full of used ones. + * + * 'freeSemaId' + 1 = 'nextSemaId': + * Can use 'freeSema' + 1 to 'freeSema'(loop back), means empty ring, + * ie, no used one. + */ + + /* MCFE semaphore id tracking ring. */ + gctUINT32 nextSemaId; + gctUINT32 freeSemaId; + + gctUINT32 semaMinThreshhold; + + /* pending semaphore id tracking ring. */ + struct + { + gctUINT32 semaId; + gctSIGNAL signal; + } + pendingSema[8]; + + gctUINT32 nextPendingPos; + gctUINT32 freePendingPos; + + /* semaphore id (array index) to handle value map. */ + gctUINT32 * semaHandleMap; + + /* + * Dirty channels, ie channels ever submitted commands. + * Need sync to (send semaphore to) system channel. + */ + gctUINT64 dirtyChannel[2]; + + /* + * Sync channels, need sync from (wait semaphore from) the system + * channel before its own jobs. + */ + gctUINT64 syncChannel[2]; + + /* Command buffer alignment. */ + gctUINT32 alignment; + + /* Commit counter. */ + gctPOINTER atomCommit; + + /* Kernel process ID. */ + gctUINT32 kernelProcessID; + +#if gcdRECORD_COMMAND + gckRECORDER recorder; +#endif + + gckFENCE fence; + + gctBOOL dummyDraw; +}; + +typedef struct _gcsEVENT * gcsEVENT_PTR; + +/* Structure holding one event to be processed. */ +typedef struct _gcsEVENT +{ + /* Pointer to next event in queue. */ + gcsEVENT_PTR next; + + /* Event information. */ + gcsHAL_INTERFACE info; + + /* Process ID owning the event. */ + gctUINT32 processID; + +#ifdef __QNXNTO__ + /* Kernel. */ + gckKERNEL kernel; +#endif + + gctBOOL fromKernel; +} +gcsEVENT; + +/* Structure holding a list of events to be processed by an interrupt. */ +typedef struct _gcsEVENT_QUEUE * gcsEVENT_QUEUE_PTR; +typedef struct _gcsEVENT_QUEUE +{ + /* Time stamp. */ + gctUINT64 stamp; + + /* Source of the event. */ + gceKERNEL_WHERE source; + + /* Pointer to head of event queue. */ + gcsEVENT_PTR head; + + /* Pointer to tail of event queue. */ + gcsEVENT_PTR tail; + + /* Next list of events. */ + gcsEVENT_QUEUE_PTR next; + + /* Current commit stamp. */ + gctUINT64 commitStamp; +} +gcsEVENT_QUEUE; + +/* + gcdREPO_LIST_COUNT defines the maximum number of event queues with different + hardware module sources that may coexist at the same time. Only two sources + are supported - gcvKERNEL_COMMAND and gcvKERNEL_PIXEL. gcvKERNEL_COMMAND + source is used only for managing the kernel command queue and is only issued + when the current command queue gets full. Since we commit event queues every + time we commit command buffers, in the worst case we can have up to three + pending event queues: + - gcvKERNEL_PIXEL + - gcvKERNEL_COMMAND (queue overflow) + - gcvKERNEL_PIXEL +*/ +#define gcdREPO_LIST_COUNT 3 + + +/* gckEVENT object. */ +struct _gckEVENT +{ + /* The object. */ + gcsOBJECT object; + + /* Pointer to required objects. */ + gckOS os; + gckKERNEL kernel; + + /* Pointer to COMMAND object, either one of the 3. */ + gckCOMMAND command; + + /* Submit function pointer, different for different command module. */ + gceSTATUS (* submitEvent)(gckEVENT, gctBOOL, gctBOOL); + + /* Time stamp. */ + gctUINT64 stamp; + gctUINT32 lastCommitStamp; + + /* Queue mutex. */ + gctPOINTER eventQueueMutex; + + /* Array of event queues. */ + gcsEVENT_QUEUE queues[29]; + gctINT32 freeQueueCount; + gctUINT8 lastID; + + /* Pending events. */ + gctPOINTER pending; + + /* List of free event structures and its mutex. */ + gcsEVENT_PTR freeEventList; + gctSIZE_T freeEventCount; + gctPOINTER freeEventMutex; + + /* Event queues. */ + gcsEVENT_QUEUE_PTR queueHead; + gcsEVENT_QUEUE_PTR queueTail; + gcsEVENT_QUEUE_PTR freeList; + gcsEVENT_QUEUE repoList[gcdREPO_LIST_COUNT]; + gctPOINTER eventListMutex; + + gctPOINTER submitTimer; + +#if gcdINTERRUPT_STATISTIC + gctPOINTER interruptCount; +#endif + + gctINT notifyState; +}; + +/* Construct a new gckEVENT object. */ +gceSTATUS +gckEVENT_Construct( + IN gckKERNEL Kernel, + IN gckCOMMAND Command, + OUT gckEVENT * Event + ); + +/* Destroy an gckEVENT object. */ +gceSTATUS +gckEVENT_Destroy( + IN gckEVENT Event + ); + +/* Reserve the next available hardware event. */ +gceSTATUS +gckEVENT_GetEvent( + IN gckEVENT Event, + IN gctBOOL Wait, + OUT gctUINT8 * EventID, + IN gceKERNEL_WHERE Source + ); + +/* Add a new event to the list of events. */ +gceSTATUS +gckEVENT_AddList( + IN gckEVENT Event, + IN gcsHAL_INTERFACE_PTR Interface, + IN gceKERNEL_WHERE FromWhere, + IN gctBOOL AllocateAllowed, + IN gctBOOL FromKernel + ); + +/* Schedule a FreeVideoMemory event. */ +gceSTATUS +gckEVENT_FreeVideoMemory( + IN gckEVENT Event, + IN gcuVIDMEM_NODE_PTR VideoMemory, + IN gceKERNEL_WHERE FromWhere + ); + +/* Schedule a signal event. */ +gceSTATUS +gckEVENT_Signal( + IN gckEVENT Event, + IN gctSIGNAL Signal, + IN gceKERNEL_WHERE FromWhere + ); + +/* Schedule an Unlock event. */ +gceSTATUS +gckEVENT_Unlock( + IN gckEVENT Event, + IN gceKERNEL_WHERE FromWhere, + IN gctPOINTER Node + ); + +gceSTATUS +gckEVENT_CommitDone( + IN gckEVENT Event, + IN gceKERNEL_WHERE FromWhere, + IN gckCONTEXT Context + ); + +gceSTATUS +gckEVENT_Submit( + IN gckEVENT Event, + IN gctBOOL Wait, + IN gctBOOL FromPower + ); + +gceSTATUS +gckEVENT_Commit( + IN gckEVENT Event, + IN gcsQUEUE_PTR Queue, + IN gctBOOL Forced + ); + +/* Event callback routine. */ +gceSTATUS +gckEVENT_Notify( + IN gckEVENT Event, + IN gctUINT32 IDs + ); + +/* Event callback routine. */ +gceSTATUS +gckEVENT_Interrupt( + IN gckEVENT Event, + IN gctUINT32 IDs + ); + +gceSTATUS +gckEVENT_Dump( + IN gckEVENT Event + ); + +/* Free all events belonging to a process. */ +gceSTATUS +gckEVENT_FreeProcess( + IN gckEVENT Event, + IN gctUINT32 ProcessID + ); + +/* gcuVIDMEM_NODE structure. */ +typedef union _gcuVIDMEM_NODE +{ + /* Allocated from gckVIDMEM. */ + struct _gcsVIDMEM_NODE_VIDMEM + { + /* Owner of this node. */ + gckVIDMEM parent; + + /* Dual-linked list of nodes. */ + gcuVIDMEM_NODE_PTR next; + gcuVIDMEM_NODE_PTR prev; + + /* Dual linked list of free nodes. */ + gcuVIDMEM_NODE_PTR nextFree; + gcuVIDMEM_NODE_PTR prevFree; + + /* Information for this node. */ + gctSIZE_T offset; + + gctUINT32 address; + gctSIZE_T bytes; + gctUINT32 alignment; + + /* Client virtual address. */ + gctPOINTER logical; + + /* Process ID owning this memory. */ + gctUINT32 processID; + + /* Locked counter. */ + gctINT32 locked; + + /* Memory pool. */ + gcePOOL pool; + + /* Kernel virtual address. */ + gctPOINTER kvaddr; + + /* mdl record pointer. */ + gctPHYS_ADDR physical; + + } + VidMem; + + /* Allocated from gckOS. */ + struct _gcsVIDMEM_NODE_VIRTUAL + { + /* Pointer to gckKERNEL object. */ + gckKERNEL kernel; + + /* Information for this node. */ + /* Contiguously allocated? */ + gctBOOL contiguous; + /* mdl record pointer... a kmalloc address. Process agnostic. */ + gctPHYS_ADDR physical; + gctSIZE_T bytes; + + /* do_mmap_pgoff address... mapped per-process. */ + gctPOINTER logical; + + /* Kernel virtual address. */ + gctPOINTER kvaddr; + + + /* Customer private handle */ + gctUINT32 gid; + + /* Page table information. */ + /* Used only when node is not contiguous */ + gctSIZE_T pageCount; + + /* Used only when node is not contiguous */ + gctPOINTER pageTables[gcvHARDWARE_NUM_TYPES]; + /* Actual physical address */ + gctUINT32 addresses[gcvHARDWARE_NUM_TYPES]; + + /* Locked counter. */ + gctINT32 lockeds[gcvHARDWARE_NUM_TYPES]; + + /* MMU page size type */ + gcePAGE_TYPE pageType; + + gceVIDMEM_TYPE type; + + /* Secure GPU virtual address. */ + gctBOOL secure; + + gctBOOL onFault; + } + Virtual; + + struct _gcsVIDMEM_NODE_VIRTUAL_CHUNK + { + /* Owner of this chunk */ + gckVIDMEM_BLOCK parent; + + /* Pointer to gckKERNEL object. */ + gckKERNEL kernel; + + /* Dual-linked list of chunk. */ + gcuVIDMEM_NODE_PTR next; + gcuVIDMEM_NODE_PTR prev; + + /* Dual linked list of free chunk. */ + gcuVIDMEM_NODE_PTR nextFree; + gcuVIDMEM_NODE_PTR prevFree; + + /* Information for this chunk. */ + gctSIZE_T offset; + gctUINT32 addresses[gcvHARDWARE_NUM_TYPES]; + gctINT32 lockeds[gcvHARDWARE_NUM_TYPES]; + gctSIZE_T bytes; + + /* Mapped user logical */ + gctPOINTER logical; + + /* Kernel virtual address. */ + gctPOINTER kvaddr; + + /* Locked counter. */ + } + VirtualChunk; + +} +gcuVIDMEM_NODE; + +/* gckVIDMEM object. */ +struct _gckVIDMEM +{ + /* Object. */ + gcsOBJECT object; + + /* Pointer to gckOS object. */ + gckOS os; + + /* mdl record pointer... a kmalloc address. Process agnostic. */ + gctPHYS_ADDR physical; + + /* Information for this video memory heap. */ + gctPHYS_ADDR_T physicalBase; + gctSIZE_T bytes; + gctSIZE_T freeBytes; + gctSIZE_T minFreeBytes; + + /* caps inherit from its allocator, ~0u if allocator was not applicable. */ + gctUINT32 capability; + + /* Mapping for each type of surface. */ + gctINT mapping[gcvVIDMEM_TYPE_COUNT]; + + /* Sentinel nodes for up to 8 banks. */ + gcuVIDMEM_NODE sentinel[8]; + + /* Allocation threshold. */ + gctSIZE_T threshold; + + /* The heap mutex. */ + gctPOINTER mutex; +}; + +/* gckVIDMEM_BLOCK object. */ +typedef struct _gcsVIDMEM_BLOCK +{ + /* Object. */ + gcsOBJECT object; + + /* Pointer to gckOS object. */ + gckOS os; + + /* linked list of nodes. */ + gckVIDMEM_BLOCK next; + + /* Contiguously allocated? */ + gctBOOL contiguous; + + /* Customer private handle */ + gctUINT32 gid; + + /* mdl record pointer... a kmalloc address. Process agnostic. */ + gctPHYS_ADDR physical; + + /* Information for this video memory virtual block. */ + gctSIZE_T bytes; + gctSIZE_T freeBytes; + + /* 1M page count. */ + gctUINT32 pageCount; + + /* Gpu virtual base of this video memory heap. */ + gctUINT32 addresses[gcvHARDWARE_NUM_TYPES]; + gctPOINTER pageTables[gcvHARDWARE_NUM_TYPES]; + + /* TODO: */ + gceVIDMEM_TYPE type; + + /* Virtual chunk. */ + gcuVIDMEM_NODE node; + + gctPOINTER mutex; + + gctBOOL secure; + gctBOOL onFault; +} +gcsVIDMEM_BLOCK; + +typedef struct _gcsVIDMEM_NODE +{ + _VIV_VIDMEM_METADATA metadata; + + /* Pointer to gcuVIDMEM_NODE. */ + gcuVIDMEM_NODE_PTR node; + + /* Pointer to gckKERNEL object. */ + gckKERNEL kernel; + + /* Mutex to protect node. */ + gctPOINTER mutex; + + /* Reference count. */ + gctPOINTER reference; + + /* Name for client to import. */ + gctUINT32 name; + + /* Link in _gckDB::videoMemList. */ + gcsLISTHEAD link; + + /* dma_buf */ + gctPOINTER dmabuf; + + /* Video memory allocation type. */ + gceVIDMEM_TYPE type; + + /* Pool from which node is allocated. */ + gcePOOL pool; + + gcsFENCE_SYNC sync[gcvENGINE_GPU_ENGINE_COUNT]; + + /* For DRM usage */ + gctUINT64 timeStamp; + gckVIDMEM_NODE tsNode; + gctUINT32 tilingMode; + gctUINT32 tsMode; + gctUINT64 clearValue; + +#if gcdCAPTURE_ONLY_MODE + gctSIZE_T captureSize; + gctPOINTER captureLogical; +#endif +} +gcsVIDMEM_NODE; + +typedef struct _gcsVIDMEM_HANDLE * gckVIDMEM_HANDLE; +typedef struct _gcsVIDMEM_HANDLE +{ + /* Pointer to gckVIDMEM_NODE. */ + gckVIDMEM_NODE node; + + /* Handle for current process. */ + gctUINT32 handle; + + /* Reference count for this handle. */ + gctPOINTER reference; +} +gcsVIDMEM_HANDLE; + +typedef struct _gcsSHBUF * gcsSHBUF_PTR; +typedef struct _gcsSHBUF +{ + /* ID. */ + gctUINT32 id; + + /* Reference count. */ + gctPOINTER reference; + + /* Data size. */ + gctUINT32 size; + + /* Data. */ + gctPOINTER data; +} +gcsSHBUF; + +typedef struct _gcsCORE_INFO +{ + gceHARDWARE_TYPE type; + gceCORE core; + gckKERNEL kernel; + gctUINT chipID; +} +gcsCORE_INFO; + +typedef struct _gcsCORE_LIST +{ + gckKERNEL kernels[gcvCORE_COUNT]; + gctUINT32 num; +} +gcsCORE_LIST; + +/* A gckDEVICE is a group of cores (gckKERNEL in software). */ +typedef struct _gcsDEVICE +{ + gcsCORE_INFO coreInfoArray[gcvCORE_COUNT]; + gctUINT32 coreNum; + gcsCORE_LIST map[gcvHARDWARE_NUM_TYPES]; + gceHARDWARE_TYPE defaultHwType; + + gckOS os; + + /* Process resource database. */ + gckDB database; + + /* Same hardware type shares one MMU. */ + gckMMU mmus[gcvHARDWARE_NUM_TYPES]; + + /* Physical address of internal SRAMs. */ + gctUINT64 sRAMBases[gcvCORE_COUNT][gcvSRAM_INTER_COUNT]; + /* Internal SRAMs' size. */ + gctUINT32 sRAMSizes[gcvCORE_COUNT][gcvSRAM_INTER_COUNT]; + /* GPU/VIP virtual address of internal SRAMs. */ + gctUINT32 sRAMBaseAddresses[gcvCORE_COUNT][gcvSRAM_INTER_COUNT]; + gctBOOL sRAMPhysFaked[gcvCORE_COUNT][gcvSRAM_INTER_COUNT]; + + /* CPU physical address of external SRAMs. */ + gctUINT64 extSRAMBases[gcvSRAM_EXT_COUNT]; + /* GPU physical address of external SRAMs. */ + gctUINT64 extSRAMGPUBases[gcvSRAM_EXT_COUNT]; + /* External SRAMs' size. */ + gctUINT32 extSRAMSizes[gcvSRAM_EXT_COUNT]; + /* GPU/VIP virtual address of external SRAMs. */ + gctUINT32 extSRAMBaseAddresses[gcvSRAM_EXT_COUNT]; + /* MDL. */ + gctPHYS_ADDR extSRAMPhysical[gcvSRAM_EXT_COUNT]; + /* IntegerId. */ + gctUINT32 extSRAMGPUPhysNames[gcvSRAM_EXT_COUNT]; + + /* Show SRAM mapping info or not. */ + gctUINT showSRAMMapInfo; + + /* Mutex to make sure stuck dump for multiple cores doesn't interleave. */ + gctPOINTER stuckDumpMutex; + + /* Mutex for multi-core combine mode command submission */ + gctPOINTER commitMutex; +} +gcsDEVICE; + +/* video memory pool functions. */ +/* Construct a new gckVIDMEM object. */ +gceSTATUS +gckVIDMEM_Construct( + IN gckOS Os, + IN gctPHYS_ADDR_T PhysicalBase, + IN gctSIZE_T Bytes, + IN gctSIZE_T Threshold, + IN gctSIZE_T Banking, + OUT gckVIDMEM * Memory + ); + +/* Destroy an gckVDIMEM object. */ +gceSTATUS +gckVIDMEM_Destroy( + IN gckVIDMEM Memory + ); + + +gceSTATUS +gckVIDMEM_HANDLE_Allocate( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE Node, + OUT gctUINT32 * Handle + ); + +gceSTATUS +gckVIDMEM_HANDLE_Reference( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 Handle + ); + +gceSTATUS +gckVIDMEM_HANDLE_Dereference( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 Handle + ); + +gceSTATUS +gckVIDMEM_HANDLE_Lookup( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 Handle, + OUT gckVIDMEM_NODE * Node + ); + +gceSTATUS +gckVIDMEM_HANDLE_Lookup2( + IN gckKERNEL Kernel, + IN gcsDATABASE_PTR Database, + IN gctUINT32 Handle, + OUT gckVIDMEM_NODE * Node + ); + +/* video memory node functions. */ +gceSTATUS +gckVIDMEM_NODE_AllocateLinear( + IN gckKERNEL Kernel, + IN gckVIDMEM VideoMemory, + IN gcePOOL Pool, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN gctUINT32 Alignment, + IN gctBOOL Specified, + IN OUT gctSIZE_T * Bytes, + OUT gckVIDMEM_NODE * NodeObject + ); + +gceSTATUS +gckVIDMEM_NODE_AllocateVirtual( + IN gckKERNEL Kernel, + IN gcePOOL Pool, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN OUT gctSIZE_T * Bytes, + OUT gckVIDMEM_NODE * NodeObject + ); + +gceSTATUS +gckVIDMEM_NODE_AllocateVirtualChunk( + IN gckKERNEL Kernel, + IN gcePOOL Pool, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN OUT gctSIZE_T * Bytes, + OUT gckVIDMEM_NODE * NodeObject + ); + +gceSTATUS +gckVIDMEM_NODE_Reference( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE Node + ); + +gceSTATUS +gckVIDMEM_NODE_Dereference( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE Node + ); + +gceSTATUS +gckVIDMEM_NODE_GetReference( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctINT32 * ReferenceCount + ); + +gceSTATUS +gckVIDMEM_NODE_Lock( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctUINT32 *Address + ); + +gceSTATUS +gckVIDMEM_NODE_Unlock( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctUINT32 ProcessID, + IN OUT gctBOOL * Asynchroneous + ); + +gceSTATUS +gckVIDMEM_NODE_CleanCache( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctSIZE_T Offset, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ); + +gceSTATUS +gckVIDMEM_NODE_InvalidateCache( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctSIZE_T Offset, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ); + +gceSTATUS +gckVIDMEM_NODE_GetLockCount( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctINT32 * LockCount + ); + +gceSTATUS +gckVIDMEM_NODE_LockCPU( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctBOOL Cacheable, + IN gctBOOL FromUser, + OUT gctPOINTER * Logical + ); + +gceSTATUS +gckVIDMEM_NODE_UnlockCPU( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctUINT32 ProcessID, + IN gctBOOL FromUser, + IN gctBOOL Defer + ); + +gceSTATUS +gckVIDMEM_NODE_GetPhysical( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctUINT32 Offset, + OUT gctPHYS_ADDR_T * PhysicalAddress + ); + +gceSTATUS +gckVIDMEM_NODE_GetGid( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctUINT32 * Gid + ); + +gceSTATUS +gckVIDMEM_NODE_GetSize( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctSIZE_T * Size + ); + +gceSTATUS +gckVIDMEM_NODE_GetType( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gceVIDMEM_TYPE * Type, + OUT gcePOOL * Pool + ); + +gceSTATUS +gckVIDMEM_NODE_Export( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctINT32 Flags, + OUT gctPOINTER *DmaBuf, + OUT gctINT32 *FD + ); + +gceSTATUS +gckVIDMEM_NODE_Name( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctUINT32 * Name + ); + +gceSTATUS +gckVIDMEM_NODE_Import( + IN gckKERNEL Kernel, + IN gctUINT32 Name, + OUT gckVIDMEM_NODE * NodeObject + ); + +gceSTATUS +gckVIDMEM_NODE_GetFd( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctINT * Fd + ); + +gceSTATUS +gckVIDMEM_NODE_WrapUserMemory( + IN gckKERNEL Kernel, + IN gcsUSER_MEMORY_DESC_PTR Desc, + IN gceVIDMEM_TYPE Type, + OUT gckVIDMEM_NODE * NodeObject, + OUT gctUINT64 * Bytes + ); + +gceSTATUS +gckVIDMEM_NODE_SetCommitStamp( + IN gckKERNEL Kernel, + IN gceENGINE Engine, + IN gckVIDMEM_NODE NodeObject, + IN gctUINT64 CommitStamp + ); + +gceSTATUS +gckVIDMEM_NODE_GetCommitStamp( + IN gckKERNEL Kernel, + IN gceENGINE Engine, + IN gckVIDMEM_NODE NodeObject, + OUT gctUINT64_PTR CommitStamp + ); + +gceSTATUS +gckVIDMEM_NODE_Find( + IN gckKERNEL Kernel, + IN gctUINT32 Address, + OUT gckVIDMEM_NODE * NodeObject, + OUT gctUINT32 * Offset + ); + +gceSTATUS +gckVIDMEM_NODE_IsContiguous( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctBOOL * Contiguous + ); + +typedef struct _gcsADDRESS_AREA * gcsADDRESS_AREA_PTR; +typedef struct _gcsADDRESS_AREA +{ + /* Page table / STLB table information. */ + gctSIZE_T stlbSize; + gckVIDMEM_NODE stlbVideoMem; + gctUINT32_PTR stlbLogical; + gctUINT32 stlbEntries; + /* stlb physical address. */ + gctPHYS_ADDR_T stlbPhysical; + + /* Free entries. */ + gctUINT32 heapList; + gctBOOL freeNodes; + + gceAREA_TYPE areaType; + + gctUINT32 mappingStart; + gctUINT32 mappingEnd; + + gctUINT32_PTR mapLogical; +} +gcsADDRESS_AREA; + +/* gckMMU object. */ +struct _gckMMU +{ + /* The object. */ + gcsOBJECT object; + + /* Pointer to gckOS object. */ + gckOS os; + + /* Pointer to gckHARDWARE object. */ + gckHARDWARE hardware; + + /* The page table mutex. */ + gctPOINTER pageTableMutex; + + /* Master TLB information. */ + gctSIZE_T mtlbSize; + gckVIDMEM_NODE mtlbVideoMem; + gctUINT32_PTR mtlbLogical; + gctUINT32 mtlbEntries; + /* mtlb physical address. */ + gctPHYS_ADDR_T mtlbPhysical; + + /* memory pool used for page table */ + gcePOOL pool; + + gctPOINTER staticSTLB; + gctBOOL enabled; + + gctSIZE_T safePageSize; + gckVIDMEM_NODE safePageVideoMem; + gctPOINTER safePageLogical; + gctUINT32 safeAddress; + /* Safe page physical address. */ + gctPHYS_ADDR_T safePagePhysical; + + /* GPU physical address flat mapping area. */ + gctUINT32 gpuPhysicalRangeCount; + gcsFLAT_MAPPING_RANGE gpuPhysicalRanges[gcdMAX_FLAT_MAPPING_COUNT]; + + /* GPU virtual address flat mapping area*/ + gctUINT32 gpuAddressRangeCount; + gcsFLAT_MAPPING_RANGE gpuAddressRanges[gcdMAX_FLAT_MAPPING_COUNT]; + + /* List of hardware which uses this MMU. */ + gcsLISTHEAD hardwareList; + + struct _gckQUEUE recentFreedAddresses; + + gcsADDRESS_AREA dynamicArea1M; + gcsADDRESS_AREA dynamicArea4K; + gcsADDRESS_AREA secureArea; + + gctBOOL dynamicAreaSetuped; + + gctBOOL sRAMMapped; + + gctUINT32 contiguousBaseAddress; + gctUINT32 externalBaseAddress; + gctUINT32 internalBaseAddress; +}; + + +gceSTATUS +gckOS_CreateKernelMapping( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Offset, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Logical + ); + +gceSTATUS +gckOS_DestroyKernelMapping( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctPOINTER Logical + ); + +gceSTATUS +gckOS_GetFd( + IN gctSTRING Name, + IN gcsFDPRIVATE_PTR Private, + OUT gctINT *Fd + ); + +/******************************************************************************* +** +** gckOS_ReadMappedPointer +** +** Read pointer mapped from user pointer which returned by gckOS_MapUserPointer. +** +** INPUT: +** +** gckOS Os +** Pointer to an gckOS object. +** +** gctPOINTER Address +** Pointer returned by gckOS_MapUserPointer. +** +** gctUINT32_PTR Data +** Pointer to hold 32 bits data. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckOS_ReadMappedPointer( + IN gckOS Os, + IN gctPOINTER Address, + IN gctUINT32_PTR Data + ); + +gceSTATUS +gckKERNEL_AttachProcess( + IN gckKERNEL Kernel, + IN gctBOOL Attach + ); + +gceSTATUS +gckKERNEL_AttachProcessEx( + IN gckKERNEL Kernel, + IN gctBOOL Attach, + IN gctUINT32 PID + ); + +gceSTATUS +gckKERNEL_AllocateVideoMemory( + IN gckKERNEL Kernel, + IN gctUINT32 Alignment, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN OUT gctSIZE_T * Bytes, + IN OUT gcePOOL * Pool, + OUT gckVIDMEM_NODE * NodeObject + ); + +gceSTATUS +gckHARDWARE_QueryIdle( + IN gckHARDWARE Hardware, + OUT gctBOOL_PTR IsIdle + ); + +gceSTATUS +gckHARDWARE_WaitFence( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT64 FenceData, + IN gctUINT32 FenceAddress, + OUT gctUINT32 *Bytes + ); + +gceSTATUS +gckHARDWARE_UpdateContextID( + IN gckHARDWARE Hardware + ); + +gceSTATUS +gckHARDWARE_QueryMcfe( + IN gckHARDWARE Hardware, + OUT const gceMCFE_CHANNEL_TYPE * Channels[], + OUT gctUINT32 * Count + ); + +#if gcdSECURITY +gceSTATUS +gckKERNEL_SecurityOpen( + IN gckKERNEL Kernel, + IN gctUINT32 GPU, + OUT gctUINT32 *Channel + ); + +/* +** Close a security service channel +*/ +gceSTATUS +gckKERNEL_SecurityClose( + IN gctUINT32 Channel + ); + +/* +** Security service interface. +*/ +gceSTATUS +gckKERNEL_SecurityCallService( + IN gctUINT32 Channel, + IN OUT gcsTA_INTERFACE * Interface + ); + +gceSTATUS +gckKERNEL_SecurityStartCommand( + IN gckKERNEL Kernel + ); + +gceSTATUS +gckKERNEL_SecurityAllocateSecurityMemory( + IN gckKERNEL Kernel, + IN gctUINT32 Bytes, + OUT gctUINT32 * Handle + ); + +gceSTATUS +gckKERNEL_SecurityExecute( + IN gckKERNEL Kernel, + IN gctPOINTER Buffer, + IN gctUINT32 Bytes + ); + +gceSTATUS +gckKERNEL_SecurityMapMemory( + IN gckKERNEL Kernel, + IN gctUINT32 *PhysicalArray, + IN gctUINT32 PageCount, + OUT gctUINT32 * GPUAddress + ); + +gceSTATUS +gckKERNEL_SecurityUnmapMemory( + IN gckKERNEL Kernel, + IN gctUINT32 GPUAddress, + IN gctUINT32 PageCount + ); + +#endif + +#if gcdENABLE_TRUST_APPLICATION +gceSTATUS +gckKERNEL_SecurityOpen( + IN gckKERNEL Kernel, + IN gctUINT32 GPU, + OUT gctUINT32 *Channel + ); + +/* +** Close a security service channel +*/ +gceSTATUS +gckKERNEL_SecurityClose( + IN gctUINT32 Channel + ); + +/* +** Security service interface. +*/ +gceSTATUS +gckKERNEL_SecurityCallService( + IN gctUINT32 Channel, + IN OUT gcsTA_INTERFACE * Interface + ); + +gceSTATUS +gckKERNEL_SecurityStartCommand( + IN gckKERNEL Kernel, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ); + +gceSTATUS +gckKERNEL_SecurityMapMemory( + IN gckKERNEL Kernel, + IN gctUINT32 *PhysicalArray, + IN gctPHYS_ADDR_T Physical, + IN gctUINT32 PageCount, + OUT gctUINT32 * GPUAddress + ); + +gceSTATUS +gckKERNEL_SecurityUnmapMemory( + IN gckKERNEL Kernel, + IN gctUINT32 GPUAddress, + IN gctUINT32 PageCount + ); + +gceSTATUS +gckKERNEL_SecurityDumpMMUException( + IN gckKERNEL Kernel + ); + +gceSTATUS +gckKERNEL_ReadMMUException( + IN gckKERNEL Kernel, + IN gctUINT32_PTR MMUStatus, + IN gctUINT32_PTR MMUException + ); + +gceSTATUS +gckKERNEL_HandleMMUException( + IN gckKERNEL Kernel, + IN gctUINT32 MMUStatus, + IN gctPHYS_ADDR_T Physical, + IN gctUINT32 GPUAddres + ); +#endif + +gceSTATUS +gckKERNEL_CreateShBuffer( + IN gckKERNEL Kernel, + IN gctUINT32 Size, + OUT gctSHBUF * ShBuf + ); + +gceSTATUS +gckKERNEL_DestroyShBuffer( + IN gckKERNEL Kernel, + IN gctSHBUF ShBuf + ); + +gceSTATUS +gckKERNEL_MapShBuffer( + IN gckKERNEL Kernel, + IN gctSHBUF ShBuf + ); + +gceSTATUS +gckKERNEL_WriteShBuffer( + IN gckKERNEL Kernel, + IN gctSHBUF ShBuf, + IN gctPOINTER UserData, + IN gctUINT32 ByteCount + ); + +gceSTATUS +gckKERNEL_ReadShBuffer( + IN gckKERNEL Kernel, + IN gctSHBUF ShBuf, + IN gctPOINTER UserData, + IN gctUINT32 ByteCount, + OUT gctUINT32 * BytesRead + ); + +gceSTATUS +gckKERNEL_GetHardwareType( + IN gckKERNEL Kernel, + OUT gceHARDWARE_TYPE *Type + ); + +/******************************************************************************\ +******************************* gckCONTEXT Object ******************************* +\******************************************************************************/ + +gceSTATUS +gckCONTEXT_Construct( + IN gckOS Os, + IN gckHARDWARE Hardware, + IN gctUINT32 ProcessID, + OUT gckCONTEXT * Context + ); + +gceSTATUS +gckCONTEXT_Destroy( + IN gckCONTEXT Context + ); + +gceSTATUS +gckCONTEXT_Update( + IN gckCONTEXT Context, + IN gctUINT32 ProcessID, + IN gcsSTATE_DELTA_PTR StateDelta + ); + +gceSTATUS +gckCONTEXT_MapBuffer( + IN gckCONTEXT Context, + OUT gctUINT64 *Logicals, + OUT gctUINT32 *Bytes + ); + +void +gckQUEUE_Enqueue( + IN gckQUEUE LinkQueue, + IN gcuQUEUEDATA *Data + ); + +void +gckQUEUE_GetData( + IN gckQUEUE LinkQueue, + IN gctUINT32 Index, + OUT gcuQUEUEDATA ** Data + ); + +gceSTATUS +gckQUEUE_Allocate( + IN gckOS Os, + IN gckQUEUE Queue, + IN gctUINT32 Size + ); + +gceSTATUS +gckQUEUE_Free( + IN gckOS Os, + IN gckQUEUE Queue + ); + +/******************************************************************************\ +****************************** gckRECORDER Object ****************************** +\******************************************************************************/ +gceSTATUS +gckRECORDER_Construct( + IN gckOS Os, + IN gckHARDWARE Hardware, + OUT gckRECORDER * Recorder + ); + +gceSTATUS +gckRECORDER_Destory( + IN gckOS Os, + IN gckRECORDER Recorder + ); + +void +gckRECORDER_AdvanceIndex( + gckRECORDER Recorder, + gctUINT64 CommitStamp + ); + +void +gckRECORDER_Record( + gckRECORDER Recorder, + gctUINT8_PTR CommandBuffer, + gctUINT32 CommandBytes, + gctUINT8_PTR ContextBuffer, + gctUINT32 ContextBytes + ); + +void +gckRECORDER_Dump( + gckRECORDER Recorder + ); + +gceSTATUS +gckRECORDER_UpdateMirror( + gckRECORDER Recorder, + gctUINT32 State, + gctUINT32 Data + ); + +/******************************************************************************\ +****************************** gckCOMMAND Object ******************************* +\******************************************************************************/ + +/* Construct a new gckCOMMAND object. */ +gceSTATUS +gckCOMMAND_Construct( + IN gckKERNEL Kernel, + IN gceHW_FE_TYPE FeType, + OUT gckCOMMAND * Command + ); + +/* Destroy an gckCOMMAND object. */ +gceSTATUS +gckCOMMAND_Destroy( + IN gckCOMMAND Command + ); + +/* Acquire command queue synchronization objects. */ +gceSTATUS +gckCOMMAND_EnterCommit( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ); + +/* Release command queue synchronization objects. */ +gceSTATUS +gckCOMMAND_ExitCommit( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ); + +/* Start the command queue. */ +gceSTATUS +gckCOMMAND_Start( + IN gckCOMMAND Command + ); + +/* Stop the command queue. */ +gceSTATUS +gckCOMMAND_Stop( + IN gckCOMMAND Command + ); + +/* Commit command buffers. */ +gceSTATUS +gckCOMMAND_Commit( + IN gckCOMMAND Command, + IN gcsHAL_SUBCOMMIT * SubCommit, + IN gctUINT32 ProcessId, + IN gctBOOL Shared, + OUT gctUINT64_PTR CommitStamp, + INOUT gctBOOL *contextSwitched + ); + +/* Reserve space in the command buffer. */ +gceSTATUS +gckCOMMAND_Reserve( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes, + OUT gctPOINTER * Buffer, + OUT gctUINT32 * BufferSize + ); + +/* + * Execute reserved space in the command buffer. + * Wait link FE version. + */ +gceSTATUS +gckCOMMAND_Execute( + IN gckCOMMAND Command, + IN gctUINT32 RequstedBytes + ); + +/* + * Execute reserved space in the command buffer. + * Async FE version. + */ +gceSTATUS +gckCOMMAND_ExecuteAsync( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes + ); + +/* + * Execute reserved space in the command buffer. + * MC-FE version. + */ +gceSTATUS +gckCOMMAND_ExecuteMultiChannel( + IN gckCOMMAND Command, + IN gctBOOL Priority, + IN gctUINT32 ChannelId, + IN gctUINT32 RequstedBytes + ); + +/* Stall the command queue. */ +gceSTATUS +gckCOMMAND_Stall( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ); + +/* Attach user process. */ +gceSTATUS +gckCOMMAND_Attach( + IN gckCOMMAND Command, + OUT gckCONTEXT * Context, + OUT gctSIZE_T * MaxState, + OUT gctUINT32 * NumStates, + IN gctUINT32 ProcessID + ); + +/* Dump command buffer being executed by GPU. */ +gceSTATUS +gckCOMMAND_DumpExecutingBuffer( + IN gckCOMMAND Command + ); + +/* Detach user process. */ +gceSTATUS +gckCOMMAND_Detach( + IN gckCOMMAND Command, + IN gckCONTEXT Context + ); + +void +gcsLIST_Init( + gcsLISTHEAD_PTR Node + ); + +void +gcsLIST_Add( + gcsLISTHEAD_PTR New, + gcsLISTHEAD_PTR Head + ); + +void +gcsLIST_AddTail( + gcsLISTHEAD_PTR New, + gcsLISTHEAD_PTR Head + ); + +void +gcsLIST_Del( + gcsLISTHEAD_PTR Node + ); + +gctBOOL +gcsLIST_Empty( + gcsLISTHEAD_PTR Head + ); + +#define gcmkLIST_FOR_EACH(pos, head) \ + for (pos = (head)->next; pos != (head); pos = pos->next) + +#define gcmkLIST_FOR_EACH_SAFE(pos, n, head) \ + for (pos = (head)->next, n = pos->next; pos != (head); \ + pos = n, n = pos->next) + +gceSTATUS +gckFENCE_Create( + IN gckOS Os, + IN gckKERNEL Kernel, + OUT gckFENCE * Fence + ); + +gceSTATUS +gckFENCE_Destory( + IN gckOS Os, + OUT gckFENCE Fence + ); + +gceSTATUS +gckFENCE_Signal( + IN gckOS Os, + IN gckFENCE Fence + ); + +gceSTATUS +gckDEVICE_Construct( + IN gckOS Os, + OUT gckDEVICE * Device + ); + +gceSTATUS +gckDEVICE_AddCore( + IN gckDEVICE Device, + IN gceCORE Core, + IN gctUINT chipID, + IN gctPOINTER Context, + IN gckKERNEL * Kernel + ); + +gceSTATUS +gckDEVICE_Destroy( + IN gckOS Os, + IN gckDEVICE Device + ); + +gceSTATUS +gckDEVICE_Dispatch( + IN gckDEVICE Device, + IN gcsHAL_INTERFACE_PTR Interface + ); + +gceSTATUS +gckDEVICE_GetMMU( + IN gckDEVICE Device, + IN gceHARDWARE_TYPE Type, + IN gckMMU *Mmu + ); + +gceSTATUS +gckDEVICE_SetMMU( + IN gckDEVICE Device, + IN gceHARDWARE_TYPE Type, + IN gckMMU Mmu + ); + +#if gcdENABLE_TRUST_APPLICATION +gceSTATUS +gckKERNEL_MapInTrustApplicaiton( + IN gckKERNEL Kernel, + IN gctPOINTER Logical, + IN gctPHYS_ADDR Physical, + IN gctUINT32 GPUAddress, + IN gctSIZE_T PageCount + ); +#endif + +#if gcdSECURITY || gcdENABLE_TRUST_APPLICATION +gceSTATUS +gckOS_OpenSecurityChannel( + IN gckOS Os, + IN gceCORE Core, + OUT gctUINT32 *Channel + ); + +gceSTATUS +gckOS_CloseSecurityChannel( + IN gctUINT32 Channel + ); + +gceSTATUS +gckOS_CallSecurityService( + IN gctUINT32 Channel, + IN gcsTA_INTERFACE * Interface + ); + +gceSTATUS +gckOS_InitSecurityChannel( + OUT gctUINT32 Channel + ); + +gceSTATUS +gckOS_AllocatePageArray( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T PageCount, + OUT gctPOINTER * PageArrayLogical, + OUT gctPHYS_ADDR * PageArrayPhysical + ); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __gc_hal_kernel_h_ */
diff --git a/hal/kernel/gc_hal_kernel_command.c b/hal/kernel/gc_hal_kernel_command.c new file mode 100644 index 0000000..684e397 --- /dev/null +++ b/hal/kernel/gc_hal_kernel_command.c
@@ -0,0 +1,4509 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" +#include "gc_hal_kernel_context.h" + +#define _GC_OBJ_ZONE gcvZONE_COMMAND + +/******************************************************************************\ +********************************* Support Code ********************************* +\******************************************************************************/ + +/******************************************************************************* +** +** _NewQueue +** +** Allocate a new command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctBOOL Stalled +** Indicate if hardware is stalled already. +** +** OUTPUT: +** +** gckCOMMAND Command +** gckCOMMAND object has been updated with a new command queue. +*/ +static gceSTATUS +_NewQueue( + IN OUT gckCOMMAND Command, + IN gctBOOL Stalled + ) +{ + gceSTATUS status; + gctINT currentIndex, newIndex; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Switch to the next command buffer. */ + currentIndex = Command->index; + newIndex = (currentIndex + 1) % gcdCOMMAND_QUEUES; + + /* Wait for availability. */ + gcmkDUMP(Command->os, "#[kernel.waitsignal]"); + + gcmkONERROR(gckOS_WaitSignal( + Command->os, + Command->queues[newIndex].signal, + gcvFALSE, + gcvINFINITE + )); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + if (newIndex < currentIndex) + { + Command->wrapCount += 1; + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_COMMAND, + 2 * 4, + "%s(%d): queue array wrapped around.\n", + __FUNCTION__, __LINE__ + ); + } + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_COMMAND, + 3 * 4, + "%s(%d): total queue wrap arounds %d.\n", + __FUNCTION__, __LINE__, Command->wrapCount + ); + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_COMMAND, + 3 * 4, + "%s(%d): switched to queue %d.\n", + __FUNCTION__, __LINE__, newIndex + ); +#endif + + /* Update gckCOMMAND object with new command queue. */ + Command->index = newIndex; + Command->newQueue = gcvTRUE; + Command->videoMem = Command->queues[newIndex].videoMem; + Command->logical = Command->queues[newIndex].logical; + Command->address = Command->queues[newIndex].address; + Command->offset = 0; + + if (currentIndex != -1) + { + if (Stalled) + { + gckOS_Signal( + Command->os, + Command->queues[currentIndex].signal, + gcvTRUE + ); + } + else + { + /* Mark the command queue as available. */ + gcmkONERROR(gckEVENT_Signal( + Command->kernel->eventObj, + Command->queues[currentIndex].signal, + gcvKERNEL_COMMAND + )); + } + } + + /* Success. */ + gcmkFOOTER_ARG("Command->index=%d", Command->index); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_IncrementCommitAtom( + IN gckCOMMAND Command, + IN gctBOOL Increment + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctINT32 atomValue; + gctBOOL powerAcquired = gcvFALSE; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Extract the gckHARDWARE and gckEVENT objects. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Grab the power mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, hardware->powerMutex, gcvINFINITE + )); + powerAcquired = gcvTRUE; + + /* Increment the commit atom. */ + if (Increment) + { + gcmkONERROR(gckOS_AtomIncrement( + Command->os, Command->atomCommit, &atomValue + )); + } + else + { + gcmkONERROR(gckOS_AtomDecrement( + Command->os, Command->atomCommit, &atomValue + )); + } + + /* Release the power mutex. */ + gcmkONERROR(gckOS_ReleaseMutex( + Command->os, hardware->powerMutex + )); + powerAcquired = gcvFALSE; + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (powerAcquired) + { + /* Release the power mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex( + Command->os, hardware->powerMutex + )); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_CheckFlushMMU( + IN gckCOMMAND Command, + IN gckHARDWARE Hardware + ) +{ +#if gcdSECURITY + return gcvSTATUS_OK; +#else + gceSTATUS status; + gctUINT32 oldValue; + gctBOOL pause = gcvFALSE; + + gctUINT8_PTR pointer; + gctUINT32 address; + gctUINT32 eventBytes; + gctUINT32 endBytes; + gctUINT32 bufferSize; + gctUINT32 executeBytes; + + gckOS_AtomicExchange(Command->os, + Hardware->pageTableDirty[gcvENGINE_RENDER], + 0, + &oldValue); + + if (oldValue) + { + /* Page Table is upated, flush mmu before commit. */ + gctUINT32 flushBytes; + + gcmkONERROR(gckHARDWARE_FlushMMU( + Hardware, + gcvNULL, + gcvINVALID_ADDRESS, + 0, + &flushBytes + )); + + gcmkONERROR(gckCOMMAND_Reserve( + Command, + flushBytes, + (gctPOINTER *)&pointer, + &bufferSize + )); + + /* Pointer to reserved address. */ + address = Command->address + Command->offset; + + /* + * subsequent 8 bytes are wait-link commands. + * Set more existed bytes for now. + */ + gcmkONERROR(gckHARDWARE_FlushMMU( + Hardware, + pointer, + address, + (bufferSize - flushBytes), + &flushBytes + )); + + gcmkONERROR(gckCOMMAND_Execute(Command, flushBytes)); + + if ((oldValue & gcvPAGE_TABLE_DIRTY_BIT_FE) + && (!Hardware->stallFEPrefetch) + ) + { + pause = gcvTRUE; + } + } + + if (pause) + { + /* Query size. */ + gcmkONERROR(gckWLFE_Event(Hardware, gcvNULL, 0, gcvKERNEL_PIXEL, &eventBytes)); + gcmkONERROR(gckWLFE_End(Hardware, gcvNULL, ~0U, &endBytes)); + + executeBytes = eventBytes + endBytes; + + /* Reserve space. */ + gcmkONERROR(gckCOMMAND_Reserve( + Command, + executeBytes, + (gctPOINTER *)&pointer, + &bufferSize + )); + + /* Pointer to reserved address. */ + address = Command->address + Command->offset; + + /* Append EVENT(29). */ + gcmkONERROR(gckWLFE_Event( + Hardware, + pointer, + 29, + gcvKERNEL_PIXEL, + &eventBytes + )); + + /* Append END. */ + pointer += eventBytes; + address += eventBytes; + + gcmkONERROR(gckWLFE_End(Hardware, pointer, address, &endBytes)); + + gcmkONERROR(gckCOMMAND_Execute(Command, executeBytes)); + } + + return gcvSTATUS_OK; +OnError: + return status; +#endif +} + +/* WaitLink FE only. */ +static gceSTATUS +_DummyDraw( + IN gckCOMMAND Command + ) +{ +#if gcdSECURITY + return gcvSTATUS_OK; +#else + gceSTATUS status; + gckHARDWARE hardware = Command->kernel->hardware; + + gctUINT8_PTR pointer; + gctUINT32 bufferSize; + + gctUINT32 dummyDrawBytes; + gceDUMMY_DRAW_TYPE dummyDrawType = gcvDUMMY_DRAW_INVALID; + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_NEED_DUMMYDRAW)) + { + dummyDrawType = gcvDUMMY_DRAW_GC400; + } + + if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_DEFER_FILL_FIX) && + gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC)) + { + dummyDrawType = gcvDUMMY_DRAW_V60; + } + + if (dummyDrawType != gcvDUMMY_DRAW_INVALID) + { + gckHARDWARE_DummyDraw(hardware, gcvNULL, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); + + /* Reserve space. */ + gcmkONERROR(gckCOMMAND_Reserve( + Command, + dummyDrawBytes, + (gctPOINTER *)&pointer, + &bufferSize + )); + + gckHARDWARE_DummyDraw(hardware, pointer, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); + + gcmkONERROR(gckCOMMAND_Execute(Command, dummyDrawBytes)); + } + + return gcvSTATUS_OK; +OnError: + return status; +#endif +} + +/* + * Wait pending semaphores. + * + * next == free: full, no more semaphores. + * (free + 1) = next: empty + */ +static gcmINLINE gceSTATUS +_WaitPendingMcfeSema( + gckCOMMAND Command + ) +{ + gceSTATUS status; + const gctUINT count = gcmCOUNTOF(Command->pendingSema); + gctUINT32 nextFreePos; + gctUINT32 timeout = gcvINFINITE; + + gcmkHEADER_ARG("freePendingPos=%u nextPendingPos=%u", + Command->freePendingPos, Command->nextPendingPos); + + nextFreePos = (Command->freePendingPos + 1) % count; + + if (nextFreePos == Command->nextPendingPos) + { + /* No pendings. */ + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + + while (nextFreePos != Command->nextPendingPos) + { + /* Timeout is infinite in the first to at least free one slot. */ + status = gckOS_WaitSignal(Command->os, + Command->pendingSema[nextFreePos].signal, + gcvFALSE, + timeout); + + if (status == gcvSTATUS_TIMEOUT) + { + /* Timeout out is OK for later pendings. */ + break; + } + + gcmkONERROR(status); + + /* Do not wait for the later slots. */ + timeout = 0; + + /* More free semaphores can be used. */ + Command->freeSemaId = Command->pendingSema[nextFreePos].semaId; + + /* Advance free pos. */ + Command->freePendingPos = nextFreePos; + nextFreePos = (nextFreePos + 1) % count; + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +static gctUINT32 +_GetFreeMcfeSemaNum( + gckCOMMAND Command + ) +{ + gctUINT32 num = 0; + + if (Command->nextSemaId <= Command->freeSemaId) + { + num = Command->freeSemaId - Command->nextSemaId; + } + else + { + num = Command->totalSemaId - Command->nextSemaId + Command->freeSemaId; + } + + return num; +} + +/* + * Get next semaphore id in semaphore ring. + * + * There are semaMinThreshhold semaphores are reserved for system operations, + * such as _SyncToSystemChannel etc. + * The rest semaphores are regular ones. + */ +static gcmINLINE gceSTATUS +_GetNextMcfeSemaId( + gckCOMMAND Command, + gctBOOL regularSema, + gctUINT32 * SemaId + ) +{ + gctUINT32 freeSemaNum = 0; + + gceSTATUS status = gcvSTATUS_OK; + + /* + * See the comments in struct definition. + * wait when run out of semaphores. + */ + freeSemaNum = _GetFreeMcfeSemaNum(Command); + + if ((regularSema && (freeSemaNum <= Command->semaMinThreshhold)) || + (!regularSema && (freeSemaNum == 0))) + { + gcmkONERROR(_WaitPendingMcfeSema(Command)); + } + + gcmkASSERT(Command->nextSemaId != Command->freeSemaId); + + /* Output the semaphore ID. */ + *SemaId = Command->nextSemaId; + + /* Advance to next. */ + if (++Command->nextSemaId == Command->totalSemaId) + { + Command->nextSemaId = 0; + } + +OnError: + return status; +} + +/* + * Get next pending pos in pending semaphore tracking ring. + */ +static gcmINLINE gceSTATUS +_GetNextPendingPos( + gckCOMMAND Command, + gctUINT32 * Pos + ) +{ + gceSTATUS status = gcvSTATUS_OK; + + /* Wait when out of pending ring. */ + if (Command->nextPendingPos == Command->freePendingPos) + { + /* Run out of pending semaphore tracking ring. */ + gcmkONERROR(_WaitPendingMcfeSema(Command)); + } + + gcmkASSERT(Command->nextPendingPos != Command->freePendingPos); + + *Pos = Command->nextPendingPos; + + /* Advance to next. */ + if (++Command->nextPendingPos == gcmCOUNTOF(Command->pendingSema)) + { + Command->nextPendingPos = 0; + } + +OnError: + return status; +} + +/* + * Sync specific channels to system channel. + * Record semaphores to pendingSema structure. + * 'SyncChannel' is cleared upon function return. + */ +static gceSTATUS +_SyncToSystemChannel( + gckCOMMAND Command, + gctUINT64 SyncChannel[2] + ) +{ + gceSTATUS status; + gckKERNEL kernel = Command->kernel; + gckHARDWARE hardware = kernel->hardware; + gctUINT8 semaId[128]; + gctUINT32 semaCount = 0; + gctUINT32 reqBytes = 0; + gctUINT32 bytes = 0; + gctUINT8_PTR buffer; + gctUINT32 i; + gctUINT32 pri; + + /* Ignore system channel. */ + SyncChannel[0] &= ~((gctUINT64)1ull); + SyncChannel[1] &= ~((gctUINT64)1ull); + + if (!SyncChannel[0] && !SyncChannel[1]) + { + return gcvSTATUS_OK; + } + + /* Query SendSemaphore command size. */ + gckMCFE_SendSemaphore(hardware, gcvNULL, 0, &reqBytes); + + for (pri = 0; pri < 2; pri++) + { + for (i = 1; i < 64 && SyncChannel[pri]; i++) + { + gctUINT32 id; + + if (!(SyncChannel[pri] & (1ull << i))) + { + continue; + } + + /* Get a free semaphore id. */ + gcmkONERROR(_GetNextMcfeSemaId(Command, gcvFALSE, &id)); + semaId[semaCount++] = (gctUINT8)id; + + gcmkONERROR(gckCOMMAND_Reserve(Command, reqBytes, (gctPOINTER *)&buffer, &bytes)); + + /* Send semaphore executed in specified channel. */ + gckMCFE_SendSemaphore(hardware, buffer, id, &bytes); + + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, pri, i, reqBytes)); + + /* Remove the sync'ed channel. */ + SyncChannel[pri] &= ~(1ull << i); + } + } + + if (semaCount > 0) + { + gctUINT32 pos = 0; + gckEVENT eventObj = kernel->eventObj; + gctUINT32 bufferLen = 0; + + /* Query WaitSemaphore command size. */ + gckMCFE_WaitSemaphore(hardware, gcvNULL, 0, &reqBytes); + reqBytes *= semaCount; + + gcmkONERROR(gckCOMMAND_Reserve(Command, reqBytes, (gctPOINTER *)&buffer, &bufferLen)); + + for (i = 0; i < semaCount; i++) + { + bytes = bufferLen; + + /* Wait semaphores executed in fixed system channel. */ + gckMCFE_WaitSemaphore(hardware, buffer, semaId[i], &bytes); + + buffer += bytes; + bufferLen -= bytes; + } + + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, 0, 0, reqBytes)); + + /* Now upload the pending semaphore tracking ring. */ + gcmkONERROR(_GetNextPendingPos(Command, &pos)); + + /* Update latest pending semaphore id. */ + Command->pendingSema[pos].semaId = (gctUINT32)semaId[semaCount - 1]; + + /* Send the signal by event. */ + gcmkONERROR(gckEVENT_Signal( + eventObj, + Command->pendingSema[pos].signal, + gcvKERNEL_PIXEL + )); + + gcmkONERROR(gckEVENT_Submit( + eventObj, + gcvTRUE, + gcvFALSE + )); + } + + return gcvSTATUS_OK; + +OnError: + return status; +} + +static gcmINLINE gceSTATUS +_SyncFromSystemChannel( + gckCOMMAND Command, + gctBOOL Priority, + gctUINT32 ChannelId + ) +{ + gceSTATUS status; + gckHARDWARE hardware = Command->kernel->hardware; + gctUINT32 reqBytes = 0; + gctUINT32 bytes = 0; + gctUINT8_PTR buffer; + gctUINT32 id; + + if (!(Command->syncChannel[Priority ? 1 : 0] & (1ull << ChannelId))) + { + /* No need to sync. */ + return gcvSTATUS_OK; + } + + /* Get a semaphore. */ + gcmkONERROR(_GetNextMcfeSemaId(Command, gcvFALSE, &id)); + + /* Send the semaphore in system channel. */ + gckMCFE_SendSemaphore(hardware, gcvNULL, 0, &reqBytes); + + gcmkONERROR(gckCOMMAND_Reserve( + Command, + reqBytes, + (gctPOINTER *)&buffer, + &bytes + )); + + gckMCFE_SendSemaphore(hardware, buffer, id, &bytes); + + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, gcvFALSE, 0, reqBytes)); + + /* Wait the semaphore in specific channel. */ + gckMCFE_WaitSemaphore(hardware, gcvNULL, 0, &reqBytes); + + gcmkONERROR(gckCOMMAND_Reserve( + Command, + reqBytes, + (gctPOINTER *)&buffer, + &bytes + )); + + gckMCFE_WaitSemaphore(hardware, buffer, id, &bytes); + + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel( + Command, + Priority, + ChannelId, + reqBytes + )); + + /* Clear the sync flag. */ + Command->syncChannel[Priority ? 1 : 0] &= ~(1ull << ChannelId); + + /* Can not track the semaphore here. */ + return gcvSTATUS_OK; + +OnError: + return status; +} + +static gceSTATUS +_CheckFlushMcfeMMU( + IN gckCOMMAND Command, + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 oldValue; + gctUINT32 reqBytes; + gctUINT32 bytes; + gctUINT8_PTR buffer; + gctUINT32 id = 0; + + gckOS_AtomicExchange(Command->os, + Hardware->pageTableDirty[gcvENGINE_RENDER], + 0, + &oldValue); + + if (!oldValue) + { + return gcvSTATUS_OK; + } + + /* + * We had sync'ed to system channel in every commit, see comments in Commit. + * It should not run into sync again here, unless there's some other place + * causes channels dirty. Let's check it here. + */ + gcmkONERROR(_SyncToSystemChannel(Command, Command->dirtyChannel)); + + /* Query flush Mcfe MMU cache command bytes. */ + gcmkONERROR(gckHARDWARE_FlushMcfeMMU(Hardware, gcvNULL, &reqBytes)); + + /* Query semaphore command bytes. */ + gcmkONERROR( + gckMCFE_SendSemaphore(Hardware, gcvNULL, 0, &bytes)); + reqBytes += bytes; + + gcmkONERROR( + gckMCFE_WaitSemaphore(Hardware, gcvNULL, 0, &bytes)); + reqBytes += bytes; + + /* Get a semaphore. */ + gcmkONERROR(_GetNextMcfeSemaId(Command, gcvFALSE, &id)); + + /* Request command buffer for system channel. */ + gcmkONERROR(gckCOMMAND_Reserve( + Command, + reqBytes, + (gctPOINTER *)&buffer, + &bytes + )); + + /* Do flush mmu. */ + gckHARDWARE_FlushMcfeMMU(Hardware, buffer, &bytes); + buffer += bytes; + + /* Send and wait semaphore in the system channel itself. */ + gcmkONERROR(gckMCFE_SendSemaphore(Hardware, buffer, id, &bytes)); + buffer += bytes; + + gcmkONERROR(gckMCFE_WaitSemaphore(Hardware, buffer, id, &bytes)); + + /* Execute flush mmu and send semaphores. */ + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, 0, 0, reqBytes)); + + /* Need sync from system channel. */ + Command->syncChannel[0] = ~1ull; + Command->syncChannel[1] = ~1ull; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +/* + * Find sema id from the map. + * Returns semaphore Id, ie the array index of semaHandleMap. + * -1 if not found. + */ +static gcmINLINE gctINT32 +_FindSemaIdFromMap( + IN gckCOMMAND Command, + IN gctUINT32 SemaHandle + ) +{ + gctUINT32 semaId = Command->nextSemaId; + + do + { + /* + * Only need to check semaId between signaledId (inclusive) and + * nextSemaId (exclusive). + */ + semaId = (semaId == 0) ? (Command->totalSemaId - 1) : (semaId - 1); + + if (Command->semaHandleMap[semaId] == SemaHandle) + { + return (gctINT32)semaId; + } + } + while (semaId != Command->freeSemaId); + + return -1; +} + +/* Put together patch list handling variables. */ +typedef struct _gcsPATCH_LIST_VARIABLE +{ + /* gcvHAL_PATCH_VIDMEM_TIMESTAMP. */ + gctUINT64 maxAsyncTimestamp; + + /* gcvHAL_PATCH_MCFE_SEMAPHORE. */ + gctBOOL semaUsed; +} +gcsPATCH_LIST_VARIABLE; + +/* Patch item handler typedef. */ +typedef gceSTATUS +(* PATCH_ITEM_HANDLER)( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gctPOINTER Patch, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ); + +static const gctUINT32 _PatchItemSize[] = +{ + 0, + (gctUINT32)sizeof(gcsHAL_PATCH_VIDMEM_ADDRESS), + (gctUINT32)sizeof(gcsHAL_PATCH_MCFE_SEMAPHORE), + (gctUINT32)sizeof(gcsHAL_PATCH_VIDMEM_TIMESTAMP), +}; + +static gceSTATUS +_HandleVidmemAddressPatch( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gctPOINTER Patch, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gcsHAL_PATCH_VIDMEM_ADDRESS * patch = Patch; + + gcmkHEADER_ARG("Command=%p location=0x%x node=0x%x offset=%x", + Command, patch->location, patch->node, patch->offset); + + (void)status; + (void)patch; + + gcmkFOOTER(); + return gcvSTATUS_OK; +} + +static gceSTATUS +_HandleMCFESemaphorePatch( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gctPOINTER Patch, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gckHARDWARE hardware = Command->kernel->hardware; + gctINT32 index; + gctUINT32 semaId; + gceSTATUS status; + gctUINT32 bytes = 8; + gctUINT32 buffer[2]; + gctUINT8_PTR location; + gcsHAL_PATCH_MCFE_SEMAPHORE * patch = (gcsHAL_PATCH_MCFE_SEMAPHORE *)Patch; + + gcmkHEADER_ARG("Command=%p location=0x%x semaHandle=%d", + Command, patch->location, patch->semaHandle); + + index = _FindSemaIdFromMap(Command, patch->semaHandle); + + if (index < 0) + { + status = _GetNextMcfeSemaId(Command, gcvTRUE, &semaId); + + if (gcmIS_ERROR(status)) + { + gcmkONERROR(_SyncToSystemChannel(Command, Command->dirtyChannel)); + gcmkONERROR(_GetNextMcfeSemaId(Command, gcvTRUE, &semaId)); + } + + Command->semaHandleMap[semaId] = patch->semaHandle; + } + else + { + semaId = (gctUINT32)index; + + /* One send must match one wait, will assign new id next time. */ + Command->semaHandleMap[semaId] = 0; + } + + if (patch->sendSema) + { + gcmkONERROR(gckMCFE_SendSemaphore(hardware, buffer, semaId, &bytes)); + } + else + { + gcmkONERROR(gckMCFE_WaitSemaphore(hardware, buffer, semaId, &bytes)); + } + + gcmkASSERT(bytes == 8); + + location = gcmUINT64_TO_PTR(CommandBuffer->logical + patch->location); + + /* Patch the command buffer. */ + gckOS_WriteMemory(Command->os, location, buffer[0]); + gckOS_WriteMemory(Command->os, location + 4, buffer[1]); + + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_HandleTimestampPatch( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gctPOINTER Patch, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gceSTATUS status; + gctUINT32 processID; + gckVIDMEM_NODE videoMem = gcvNULL; + gcsHAL_PATCH_VIDMEM_TIMESTAMP * patch = Patch; + gceENGINE engine = Command->feType == gcvHW_FE_ASYNC ? gcvENGINE_BLT + : gcvENGINE_RENDER; + + gcmkHEADER_ARG("Command=%p node=0x%x", Command, patch->handle); + + /* Get the current process ID. */ + gcmkONERROR(gckOS_GetProcessID(&processID)); + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Command->kernel, + processID, + patch->handle, + &videoMem)); + + gcmkVERIFY_OK(gckVIDMEM_NODE_Reference(Command->kernel, videoMem)); + + /* Touch video memory node. */ + gcmkVERIFY_OK( + gckVIDMEM_NODE_SetCommitStamp(Command->kernel, + engine, + videoMem, + Command->commitStamp)); + + if ((engine == gcvENGINE_RENDER) && Command->kernel->asyncCommand) + { + /* Find the latest timestamp of the nodes used in async FE. */ + gctUINT64 stamp = 0; + + /* Get stamp touched async command buffer. */ + gcmkVERIFY_OK( + gckVIDMEM_NODE_GetCommitStamp(Command->kernel, + gcvENGINE_BLT, + videoMem, + &stamp)); + + /* Find latest one. */ + if (PatchListVar->maxAsyncTimestamp < stamp) + { + PatchListVar->maxAsyncTimestamp = stamp; + } + } + +OnError: + if (videoMem) + { + gckVIDMEM_NODE_Dereference(Command->kernel, videoMem); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +static gceSTATUS +_HandlePatchListSingle( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gcsHAL_PATCH_LIST * PatchList, + IN gctBOOL NeedCopy, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gceSTATUS status; + /* 256 bytes for storage. */ + gctUINT64 storage[32]; + gctPOINTER kArray = gcvNULL; + gctPOINTER userPtr = gcvNULL; + gctUINT32 index = 0; + gctUINT32 count = 0; + gctUINT32 itemSize = 0; + gctUINT32 batchCount = 0; + + static const PATCH_ITEM_HANDLER patchHandler[] = + { + gcvNULL, + _HandleVidmemAddressPatch, + _HandleMCFESemaphorePatch, + _HandleTimestampPatch, + }; + PATCH_ITEM_HANDLER handler; + + gcmkHEADER_ARG("Command=%p CommandBuffer=%p PatchList=%p type=%d", + Command, CommandBuffer, PatchList, PatchList->type); + + if (PatchList->type >= gcmCOUNTOF(_PatchItemSize) || PatchList->type >= gcmCOUNTOF(patchHandler)) + { + /* Exceeds buffer max size. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + itemSize = _PatchItemSize[PatchList->type]; + + batchCount = (gctUINT32)(sizeof(storage) / itemSize); + + handler = patchHandler[PatchList->type]; + + while (index < PatchList->count) + { + gctUINT i; + gctUINT8_PTR ptr; + + /* Determine batch count, don't handle too many in one batch. */ + count = PatchList->count - index; + + if (count > batchCount) + { + count = batchCount; + } + + userPtr = gcmUINT64_TO_PTR(PatchList->patchArray + itemSize * index); + + /* Copy/map a patch array batch from user. */ + if (NeedCopy) + { + kArray = storage; + + status = gckOS_CopyFromUserData( + Command->os, + kArray, + userPtr, + itemSize * count + ); + } + else + { + status = gckOS_MapUserPointer( + Command->os, + userPtr, + itemSize * count, + (gctPOINTER *)&kArray + ); + } + + if (gcmIS_ERROR(status)) + { + userPtr = gcvNULL; + gcmkONERROR(status); + } + + /* Advance to next batch. */ + index += count; + + ptr = (gctUINT8_PTR)kArray; + + for (i = 0; i < count; i++) + { + /* Call handler. */ + gcmkONERROR( + handler(Command, CommandBuffer, ptr, PatchListVar)); + + /* Advance to next patch. */ + ptr += itemSize; + } + + /* Unmap user pointer if mapped. */ + if (!NeedCopy) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + itemSize * count, + kArray + )); + } + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (!NeedCopy && userPtr) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + itemSize * count, + kArray + )); + + userPtr = gcvNULL; + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_HandlePatchList( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + OUT gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gceSTATUS status; + gctBOOL needCopy = gcvFALSE; + gcsHAL_PATCH_LIST storage; + gcsHAL_PATCH_LIST * kPatchList = gcvNULL; + gctPOINTER userPtr = gcmUINT64_TO_PTR(CommandBuffer->patchHead); + + gcmkHEADER_ARG("Command=%p CommandBuffer=%p", Command, CommandBuffer); + + /* Check wehther we need to copy the structures or not. */ + gcmkONERROR(gckOS_QueryNeedCopy(Command->os, 0, &needCopy)); + + while (userPtr) + { + gctUINT64 next; + + /* Copy/map a patch from user. */ + if (needCopy) + { + kPatchList = &storage; + + status = gckOS_CopyFromUserData( + Command->os, + kPatchList, + userPtr, + sizeof(gcsHAL_PATCH_LIST) + ); + } + else + { + status = gckOS_MapUserPointer( + Command->os, + userPtr, + sizeof(gcsHAL_PATCH_LIST), + (gctPOINTER *)&kPatchList + ); + } + + if (gcmIS_ERROR(status)) + { + userPtr = gcvNULL; + gcmkONERROR(status); + } + + /* Do handle patch. */ + gcmkASSERT(kPatchList->type < gcvHAL_PATCH_TYPE_COUNT); + + gcmkONERROR( + _HandlePatchListSingle(Command, + CommandBuffer, + kPatchList, + needCopy, + PatchListVar)); + + next = kPatchList->next; + + /* Unmap user pointer if mapped. */ + if (!needCopy) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + sizeof(gcsHAL_PATCH_LIST), + kPatchList + )); + } + + /* Advance to next patch from user. */ + userPtr = gcmUINT64_TO_PTR(next); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (!needCopy && userPtr) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + sizeof(gcsHAL_PATCH_LIST), + kPatchList + )); + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_WaitForAsyncCommandStamp( + IN gckCOMMAND Command, + IN gctUINT64 Stamp + ) +{ + gctUINT32 bytes; + gceSTATUS status; + gctUINT32 fenceAddress; + gctUINT32 bufferSize; + gctPOINTER pointer; + gckCOMMAND asyncCommand = Command->kernel->asyncCommand; + + gcmkHEADER_ARG("Stamp = 0x%llx", Stamp); + + if (*(gctUINT64 *)asyncCommand->fence->logical >= Stamp) + { + /* Already satisfied, skip. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + fenceAddress = asyncCommand->fence->address; + + gcmkONERROR(gckHARDWARE_WaitFence( + Command->kernel->hardware, + gcvNULL, + Stamp, + fenceAddress, + &bytes + )); + + gcmkONERROR(gckCOMMAND_Reserve( + Command, + bytes, + &pointer, + &bufferSize + )); + + gcmkONERROR(gckHARDWARE_WaitFence( + Command->kernel->hardware, + pointer, + Stamp, + fenceAddress, + &bytes + )); + + gcmkONERROR(gckCOMMAND_Execute(Command, bytes)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +/******************************************************************************\ +****************************** gckCOMMAND API Code ****************************** +\******************************************************************************/ + +/******************************************************************************* +** +** gckCOMMAND_Construct +** +** Construct a new gckCOMMAND object. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** OUTPUT: +** +** gckCOMMAND * Command +** Pointer to a variable that will hold the pointer to the gckCOMMAND +** object. +*/ +gceSTATUS +gckCOMMAND_Construct( + IN gckKERNEL Kernel, + IN gceHW_FE_TYPE FeType, + OUT gckCOMMAND * Command + ) +{ + gckOS os; + gckCOMMAND command = gcvNULL; + gceSTATUS status; + gctINT i; + gctPOINTER pointer = gcvNULL; + gctSIZE_T pageSize; + + gcmkHEADER_ARG("Kernel=%p", Kernel); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Command != gcvNULL); + + /* Extract the gckOS object. */ + os = Kernel->os; + + /* Allocate the gckCOMMAND structure. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckCOMMAND), &pointer)); + command = pointer; + + /* Reset the entire object. */ + gcmkONERROR(gckOS_ZeroMemory(command, gcmSIZEOF(struct _gckCOMMAND))); + + /* Initialize the gckCOMMAND object.*/ + command->object.type = gcvOBJ_COMMAND; + command->kernel = Kernel; + command->os = os; + + command->feType = FeType; + + /* Get the command buffer requirements. */ + gcmkONERROR(gckHARDWARE_QueryCommandBuffer( + Kernel->hardware, + gcvENGINE_RENDER, + &command->alignment, + gcvNULL, + gcvNULL + )); + + /* Create the command queue mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &command->mutexQueue)); + + /* Create the context switching mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContext)); + + /* Create the context switching mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContextSeq)); + + /* Create the power management semaphore. */ + gcmkONERROR(gckOS_CreateSemaphore(os, &command->powerSemaphore)); + + /* Create the commit atom. */ + gcmkONERROR(gckOS_AtomConstruct(os, &command->atomCommit)); + + /* Get the page size from teh OS. */ + gcmkONERROR(gckOS_GetPageSize(os, &pageSize)); + + gcmkSAFECASTSIZET(command->pageSize, pageSize); + + /* Get process ID. */ + gcmkONERROR(gckOS_GetProcessID(&command->kernelProcessID)); + + /* Set hardware to pipe 0. */ + command->pipeSelect = gcvPIPE_INVALID; + + /* Pre-allocate the command queues. */ + for (i = 0; i < gcdCOMMAND_QUEUES; ++i) + { +#if !gcdCAPTURE_ONLY_MODE + gcePOOL pool = gcvPOOL_DEFAULT; +#else + gcePOOL pool = gcvPOOL_VIRTUAL; +#endif + + gctSIZE_T size = pageSize; + gckVIDMEM_NODE videoMem = gcvNULL; + gctUINT32 allocFlag = 0; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag = gcvALLOC_FLAG_CACHEABLE; +#endif + + /* Allocate video memory node for command buffers. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + Kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + allocFlag, + &size, + &pool, + &videoMem + )); + + command->queues[i].videoMem = videoMem; + + /* Lock for GPU access. */ + gcmkONERROR(gckVIDMEM_NODE_Lock( + Kernel, + videoMem, + &command->queues[i].address + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + Kernel, + videoMem, + gcvFALSE, + gcvFALSE, + &command->queues[i].logical + )); + + gcmkONERROR(gckOS_CreateSignal( + os, gcvFALSE, &command->queues[i].signal + )); + + gcmkONERROR(gckOS_Signal( + os, command->queues[i].signal, gcvTRUE + )); + } + +#if gcdRECORD_COMMAND + gcmkONERROR(gckRECORDER_Construct(os, Kernel->hardware, &command->recorder)); +#endif + + gcmkONERROR(gckFENCE_Create( + os, Kernel, &command->fence + )); + + /* No command queue in use yet. */ + command->index = -1; + command->logical = gcvNULL; + command->newQueue = gcvFALSE; + + /* Query mcfe semaphore count. */ + if (FeType == gcvHW_FE_MULTI_CHANNEL) + { + command->totalSemaId = 128; + + /* Empty sema id ring. */ + command->nextSemaId = 0; + command->freeSemaId = command->totalSemaId - 1; + + command->semaMinThreshhold = 16; + + /* Create signals. */ + for (i = 0; i < (gctINT)gcmCOUNTOF(command->pendingSema); i++) + { + gcmkONERROR(gckOS_CreateSignal( + os, + gcvFALSE, + &command->pendingSema[i].signal + )); + } + + /* Empty pending sema tracking ring. */ + command->nextPendingPos = 0; + command->freePendingPos = gcmCOUNTOF(command->pendingSema) - 1; + + /* Allocate sema handle mapping. */ + gcmkONERROR(gckOS_Allocate( + os, + command->totalSemaId * sizeof(gctUINT32), + &pointer + )); + + command->semaHandleMap = (gctUINT32 *)pointer; + + gcmkVERIFY_OK(gckOS_ZeroMemory( + command->semaHandleMap, + command->totalSemaId * sizeof(gctUINT32) + )); + } + + /* Command is not yet running. */ + command->running = gcvFALSE; + + /* Command queue is idle. */ + command->idle = gcvTRUE; + + /* Commit stamp start from 1. */ + command->commitStamp = 1; + + command->dummyDraw = gcvTRUE; + + /* Return pointer to the gckCOMMAND object. */ + *Command = command; + + /* Success. */ + gcmkFOOTER_ARG("*Command=0x%x", *Command); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (command != gcvNULL) + { + gcmkVERIFY_OK(gckCOMMAND_Destroy(command)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Destroy +** +** Destroy an gckCOMMAND object. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to destroy. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Destroy( + IN gckCOMMAND Command + ) +{ + gctINT i; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Stop the command queue. */ + gcmkVERIFY_OK(gckCOMMAND_Stop(Command)); + + for (i = 0; i < gcdCOMMAND_QUEUES; ++i) + { + if (Command->queues[i].signal) + { + gcmkVERIFY_OK(gckOS_DestroySignal( + Command->os, Command->queues[i].signal + )); + } + + if (Command->queues[i].logical) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + Command->kernel, + Command->queues[i].videoMem, + 0, + gcvFALSE, + gcvFALSE + )); + + gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( + Command->kernel, + Command->queues[i].videoMem, + 0, + gcvNULL + )); + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + Command->kernel, + Command->queues[i].videoMem + )); + + Command->queues[i].videoMem = gcvNULL; + Command->queues[i].logical = gcvNULL; + } + } + + if (Command->mutexContext) + { + /* Delete the context switching mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContext)); + } + + if (Command->mutexContextSeq != gcvNULL) + gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContextSeq)); + + if (Command->mutexQueue) + { + /* Delete the command queue mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexQueue)); + } + + if (Command->powerSemaphore) + { + /* Destroy the power management semaphore. */ + gcmkVERIFY_OK(gckOS_DestroySemaphore(Command->os, Command->powerSemaphore)); + } + + if (Command->atomCommit) + { + /* Destroy the commit atom. */ + gcmkVERIFY_OK(gckOS_AtomDestroy(Command->os, Command->atomCommit)); + } + +#if gcdRECORD_COMMAND + gckRECORDER_Destory(Command->os, Command->recorder); +#endif + + if (Command->stateMap) + { + gcmkOS_SAFE_FREE(Command->os, Command->stateMap); + } + + if (Command->semaHandleMap) + { + gcmkOS_SAFE_FREE(Command->os, Command->semaHandleMap); + } + + if (Command->fence) + { + gcmkVERIFY_OK(gckFENCE_Destory(Command->os, Command->fence)); + } + + /* Mark object as unknown. */ + Command->object.type = gcvOBJ_UNKNOWN; + + /* Free the gckCOMMAND object. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, Command)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckCOMMAND_EnterCommit +** +** Acquire command queue synchronization objects. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to destroy. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_EnterCommit( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctBOOL atomIncremented = gcvFALSE; + gctBOOL semaAcquired = gcvFALSE; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Extract the gckHARDWARE and gckEVENT objects. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + if (!FromPower) + { + /* Increment COMMIT atom to let power management know that a commit is + ** in progress. */ + gcmkONERROR(_IncrementCommitAtom(Command, gcvTRUE)); + atomIncremented = gcvTRUE; + + /* Notify the system the GPU has a commit. */ + gcmkONERROR(gckOS_Broadcast(Command->os, + hardware, + gcvBROADCAST_GPU_COMMIT)); + + /* Acquire the power management semaphore. */ + gcmkONERROR(gckOS_AcquireSemaphore(Command->os, + Command->powerSemaphore)); + semaAcquired = gcvTRUE; + } + + /* Grab the conmmand queue mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Command->os, + Command->mutexQueue, + gcvINFINITE)); + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (semaAcquired) + { + /* Release the power management semaphore. */ + gcmkVERIFY_OK(gckOS_ReleaseSemaphore( + Command->os, Command->powerSemaphore + )); + } + + if (atomIncremented) + { + /* Decrement the commit atom. */ + gcmkVERIFY_OK(_IncrementCommitAtom( + Command, gcvFALSE + )); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_ExitCommit +** +** Release command queue synchronization objects. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to destroy. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_ExitCommit( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Release the power mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); + + if (!FromPower) + { + /* Release the power management semaphore. */ + gcmkONERROR(gckOS_ReleaseSemaphore(Command->os, + Command->powerSemaphore)); + + /* Decrement the commit atom. */ + gcmkONERROR(_IncrementCommitAtom(Command, gcvFALSE)); + } + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_StartWaitLinkFE( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctUINT32 waitOffset = 0; + gctUINT32 waitLinkBytes; + gctPOINTER logical; + gctUINT32 address; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + if (Command->running) + { + /* Command queue already running. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + /* Extract the gckHARDWARE object. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Query the size of WAIT/LINK command sequence. */ + gcmkONERROR(gckWLFE_WaitLink( + hardware, + gcvNULL, + ~0U, + Command->offset, + &waitLinkBytes, + gcvNULL, + gcvNULL + )); + + if ((Command->pageSize - Command->offset < waitLinkBytes) + || (Command->logical == gcvNULL) + ) + { + /* Start at beginning of a new queue. */ + gcmkONERROR(_NewQueue(Command, gcvTRUE)); + } + + logical = (gctUINT8_PTR) Command->logical + Command->offset; + address = Command->address + Command->offset; + + /* Append WAIT/LINK. */ + gcmkONERROR(gckWLFE_WaitLink( + hardware, + logical, + address, + 0, + &waitLinkBytes, + &waitOffset, + &Command->waitPos.size + )); + + /* Update wait command position. */ + Command->waitPos.videoMem = Command->videoMem; + Command->waitPos.offset = Command->offset + waitOffset; + Command->waitPos.logical = (gctUINT8_PTR) logical + waitOffset; + Command->waitPos.address = address + waitOffset; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + logical, + waitLinkBytes + )); + + /* Adjust offset. */ + Command->offset += waitLinkBytes; + Command->newQueue = gcvFALSE; + + gcmkDUMP(Command->os, "#[wait-link: fe start]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + logical, + address, + waitLinkBytes + ); + +#if gcdSECURITY + /* Start FE by calling security service. */ + gckKERNEL_SecurityStartCommand( + Command->kernel + ); +#else + +#if !gcdCAPTURE_ONLY_MODE + /* Enable command processor. */ + gcmkONERROR(gckWLFE_Execute( + hardware, + address, + waitLinkBytes + )); +#endif + +#endif + + /* Command queue is running. */ + Command->running = gcvTRUE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_StartAsyncFE( + IN gckCOMMAND Command + ) +{ + if ((Command->pageSize <= Command->offset) || + (Command->logical == gcvNULL)) + { + /* Start at beginning of a new queue. */ + gcmkVERIFY_OK(_NewQueue(Command, gcvTRUE)); + } + + /* Command queue is running. */ + Command->running = gcvTRUE; + + /* Nothing to do. */ + return gcvSTATUS_OK; +} + +static gceSTATUS +_StartMCFE( + IN gckCOMMAND Command + ) +{ + if ((Command->pageSize <= Command->offset) || + (Command->logical == gcvNULL)) + { + /* Start at beginning of a new queue. */ + gcmkVERIFY_OK(_NewQueue(Command, gcvTRUE)); + } + + /* Command queue is running. */ + Command->running = gcvTRUE; + + /* Nothing to do. */ + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckCOMMAND_Start +** +** Start up the command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to start. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Start( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Command=%p", Command); + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + gcmkONERROR(_StartWaitLinkFE(Command)); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + gcmkONERROR(_StartMCFE(Command)); + } + else + { + gcmkONERROR(_StartAsyncFE(Command)); + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; + +} + +static gceSTATUS +_StopWaitLinkFE( + IN gckCOMMAND Command + ) +{ + gckHARDWARE hardware; + gceSTATUS status; + gctUINT32 idle; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Extract the gckHARDWARE object. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Replace last WAIT with END. */ + gcmkONERROR(gckWLFE_End( + hardware, + Command->waitPos.logical, + Command->waitPos.address, + &Command->waitPos.size + )); + + gcmkDUMP(Command->os, "#[end: fe stop]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + Command->waitPos.logical, + Command->waitPos.address, + Command->waitPos.size + ); + +#if gcdSECURITY + gcmkONERROR(gckKERNEL_SecurityExecute( + Command->kernel, Command->waitPos.logical, 8 + )); +#endif + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail(Command->kernel->hardware, + Command->logical, + Command->offset)); + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->waitPos.videoMem, + Command->waitPos.offset, + Command->waitPos.logical, + Command->waitPos.size + )); + + /* Wait for idle. */ + gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle)); + + /* Command queue is no longer running. */ + Command->running = gcvFALSE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_StopAsyncFE( + IN gckCOMMAND Command + ) +{ + gckHARDWARE hardware; + gceSTATUS status; + gctUINT32 idle; + + gcmkHEADER_ARG("Command=%p", Command); + + hardware = Command->kernel->hardware; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail(hardware, + Command->logical, + Command->offset)); + + /* Wait for idle. */ + gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle)); + + /* Command queue is no longer running. */ + Command->running = gcvFALSE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_StopMCFE( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + + gcmkHEADER_ARG("Command=%p", Command); + + hardware = Command->kernel->hardware; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail(hardware, + Command->logical, + Command->offset)); + + /* Command queue is no longer running. */ + Command->running = gcvFALSE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Stop +** +** Stop the command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to stop. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Stop( + IN gckCOMMAND Command + ) +{ + if (!Command->running) + { + /* Command queue is not running. */ + return gcvSTATUS_OK; + } + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + return _StopWaitLinkFE(Command); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + return _StopMCFE(Command); + } + else + { + return _StopAsyncFE(Command); + } +} + +static gceSTATUS +_CommitWaitLinkOnce( + IN gckCOMMAND Command, + IN gckCONTEXT Context, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gcsSTATE_DELTA_PTR StateDelta, + IN gctUINT32 ProcessID, + IN gctBOOL Shared, + INOUT gctBOOL *contextSwitched + ) +{ + gceSTATUS status; + gctBOOL commitEntered = gcvFALSE; + gctBOOL contextAcquired = gcvFALSE; + gckHARDWARE hardware; + gcsPATCH_LIST_VARIABLE patchListVar = {0, 0}; + + gcsCONTEXT_PTR contextBuffer; + gctUINT8_PTR commandBufferLogical = gcvNULL; + gctUINT32 commandBufferAddress = 0; + gckVIDMEM_NODE commandBufferVideoMem = gcvNULL; + gctUINT8_PTR commandBufferTail = gcvNULL; + gctUINT commandBufferSize; + gctUINT32 linkBytes; + gctSIZE_T bytes; + gctUINT32 offset; + gctPOINTER entryLogical; + gctUINT32 entryAddress; + gctUINT32 entryBytes; + gctUINT32 exitAddress; + gctUINT32 exitBytes; + gctPOINTER waitLinkLogical; + gctUINT32 waitLinkAddress; + gctUINT32 waitLinkBytes; + gctUINT32 waitOffset; + gctUINT32 waitSize; + +#if gcdCAPTURE_ONLY_MODE + gctINT i; +#endif + +#ifdef __QNXNTO__ + gctPOINTER userCommandBufferLogical = gcvNULL; + gctBOOL userCommandBufferLogicalMapped = gcvFALSE; +#endif + +#if gcdDUMP_IN_KERNEL + gctPOINTER contextDumpLogical = gcvNULL; +# endif + gctUINT32 exitLinkLow = 0, exitLinkHigh = 0; + gctUINT32 entryLinkLow = 0, entryLinkHigh = 0; + gctUINT32 commandLinkLow = 0, commandLinkHigh = 0; + + gcmkHEADER_ARG("Command=%p CommandBuffer=%p ProcessID=%d", + Command, CommandBuffer, ProcessID + ); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + gcmkASSERT(Command->feType == gcvHW_FE_WAIT_LINK); + + /* Acquire the command queue. */ + gcmkONERROR(gckCOMMAND_EnterCommit(Command, gcvFALSE)); + commitEntered = gcvTRUE; + + /* Acquire the context switching mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, Command->mutexContext, gcvINFINITE + )); + contextAcquired = gcvTRUE; + + /* Extract the gckHARDWARE and gckEVENT objects. */ + hardware = Command->kernel->hardware; + + + gcmkONERROR( + _HandlePatchList(Command, CommandBuffer, &patchListVar)); + + /* Query the size of LINK command. */ + gcmkONERROR(gckWLFE_Link( + hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL + )); + + /* Compute the command buffer entry and the size. */ + commandBufferLogical + = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset; + + commandBufferAddress = CommandBuffer->address + + CommandBuffer->startOffset; + +#ifdef __QNXNTO__ + gcmkONERROR(gckVIDMEM_HANDLE_Lookup( + Command->kernel, + ProcessID, + CommandBuffer->videoMemNode, + &commandBufferVideoMem + )); + + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + Command->kernel, + commandBufferVideoMem, + gcvFALSE, + gcvFALSE, + &userCommandBufferLogical + )); + + commandBufferLogical = (gctUINT8_PTR)userCommandBufferLogical + CommandBuffer->startOffset; + userCommandBufferLogicalMapped =gcvTRUE; +#endif + + commandBufferSize = CommandBuffer->size; + + gcmkONERROR(_CheckFlushMMU(Command, hardware)); + + if (Command->dummyDraw == gcvTRUE && + Context != gcvNULL) + { + Command->dummyDraw = gcvFALSE; + gcmkONERROR(_DummyDraw(Command)); + } + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT) && + Command->kernel->asyncCommand && + patchListVar.maxAsyncTimestamp != 0) + { + gcmkONERROR(_WaitForAsyncCommandStamp( + Command, + patchListVar.maxAsyncTimestamp + )); + } + + /* Get the current offset. */ + offset = Command->offset; + + /* Compute number of bytes left in current kernel command queue. */ + bytes = Command->pageSize - offset; + + /* Query the size of WAIT/LINK command sequence. */ + gcmkONERROR(gckWLFE_WaitLink( + hardware, + gcvNULL, + ~0U, + offset, + &waitLinkBytes, + gcvNULL, + gcvNULL + )); + + /* Is there enough space in the current command queue? */ + if (bytes < waitLinkBytes) + { + /* No, create a new one. */ + gcmkONERROR(_NewQueue(Command, gcvFALSE)); + + /* Get the new current offset. */ + offset = Command->offset; + + /* Recompute the number of bytes in the new kernel command queue. */ + bytes = Command->pageSize - offset; + gcmkASSERT(bytes >= waitLinkBytes); + } + + /* Compute the location if WAIT/LINK command sequence. */ + waitLinkLogical = (gctUINT8_PTR) Command->logical + offset; + waitLinkAddress = Command->address + offset; + + /* Context switch required? */ + if (Context == gcvNULL) + { + /* See if we have to switch pipes for the command buffer. */ + if (CommandBuffer->entryPipe == (gctUINT32)(Command->pipeSelect)) + { + /* Skip reserved head bytes. */ + offset = CommandBuffer->reservedHead; + } + else + { + gctUINT32 pipeBytes = CommandBuffer->reservedHead; + + /* The current hardware and the entry command buffer pipes + ** are different, switch to the correct pipe. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, + commandBufferLogical, + CommandBuffer->entryPipe, + &pipeBytes + )); + + /* Do not skip pipe switching sequence. */ + offset = 0; + + /* Reserved bytes in userspace must be exact for a pipeSelect. */ + gcmkASSERT(pipeBytes == CommandBuffer->reservedHead); + } + + /* Compute the entry. */ + entryLogical = commandBufferLogical + offset; + entryAddress = commandBufferAddress + offset; + entryBytes = commandBufferSize - offset; + + Command->currContext = gcvNULL; + } +#if gcdDEBUG_OPTION && gcdDEBUG_FORCE_CONTEXT_UPDATE + else if (1) +#else + else if (Command->currContext != Context) +#endif + { + /* Get the current context buffer. */ + contextBuffer = Context->buffer; + + /* Yes, merge in the deltas. */ + gcmkONERROR(gckCONTEXT_Update(Context, ProcessID, StateDelta)); + + /*************************************************************** + ** SWITCHING CONTEXT. + */ + + /* Determine context buffer entry offset. */ + offset = (Command->pipeSelect == gcvPIPE_3D) + + /* Skip pipe switching sequence. */ + ? Context->entryOffset3D + Context->pipeSelectBytes + + /* Do not skip pipe switching sequence. */ + : Context->entryOffset3D; + + /* Compute the entry. */ + entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset; + entryAddress = contextBuffer->address + offset; + entryBytes = Context->bufferSize - offset; + + /* See if we have to switch pipes between the context + and command buffers. */ + if (CommandBuffer->entryPipe == gcvPIPE_3D) + { + /* Skip reserved head bytes. */ + offset = CommandBuffer->reservedHead; + } + else + { + gctUINT32 pipeBytes = CommandBuffer->reservedHead; + + /* The current hardware and the initial context pipes are + different, switch to the correct pipe. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, + commandBufferLogical, + CommandBuffer->entryPipe, + &pipeBytes + )); + + /* Do not skip pipe switching sequence. */ + offset = 0; + + /* Reserved bytes in userspace must be exact for a pipeSelect. */ + gcmkASSERT(pipeBytes == CommandBuffer->reservedHead); + } + + /* Generate a LINK from the context buffer to + the command buffer. */ + gcmkONERROR(gckWLFE_Link( + hardware, + contextBuffer->link3D, + commandBufferAddress + offset, + commandBufferSize - offset, + &linkBytes, + &commandLinkLow, + &commandLinkHigh + )); + +#if gcdCAPTURE_ONLY_MODE + for (i = 0; i < gcdCONTEXT_BUFFER_NUM; ++i) + { + gcsCONTEXT_PTR buffer = contextBuffer; + + gckOS_CopyToUserData(Command->os, buffer->logical, CommandBuffer->contextLogical[i], Context->bufferSize); + + buffer = buffer->next; + } +#endif + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + contextBuffer->videoMem, + entryAddress - contextBuffer->address, + entryLogical, + entryBytes + )); + + /* Update the current context. */ + Command->currContext = Context; + + if (contextSwitched) + { + *contextSwitched = gcvTRUE; + } + +#if gcdDUMP_IN_KERNEL + contextDumpLogical = entryLogical; +#endif + +#if gcdSECURITY + /* Commit context buffer to trust zone. */ + gckKERNEL_SecurityExecute( + Command->kernel, + entryLogical, + entryBytes - 8 + ); +#endif + +#if gcdRECORD_COMMAND + gckRECORDER_Record( + Command->recorder, + gcvNULL, + 0xFFFFFFFF, + entryLogical, + entryBytes + ); +#endif + } + + /* Same context. */ + else + { + /* See if we have to switch pipes for the command buffer. */ + if (CommandBuffer->entryPipe == (gctUINT32)(Command->pipeSelect)) + { + /* Skip reserved head bytes. */ + offset = CommandBuffer->reservedHead; + } + else + { + gctUINT32 pipeBytes = CommandBuffer->reservedHead; + + /* The current hardware and the entry command buffer pipes + ** are different, switch to the correct pipe. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, + commandBufferLogical, + CommandBuffer->entryPipe, + &pipeBytes + )); + + /* Do not skip pipe switching sequence. */ + offset = 0; + + /* Reserved bytes in userspace must be exact for a pipeSelect. */ + gcmkASSERT(pipeBytes == CommandBuffer->reservedHead); + } + + /* Compute the entry. */ + entryLogical = commandBufferLogical + offset; + entryAddress = commandBufferAddress + offset; + entryBytes = commandBufferSize - offset; + } + + (void)entryLogical; + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ + /* Some extra commands (at beginning) are required for new queue. */ + exitAddress = Command->address; + exitBytes = Command->offset + waitLinkBytes; + } + else + { + /* Still within the preexisting command queue, jump to the new + WAIT/LINK command sequence. */ + exitAddress = waitLinkAddress; + exitBytes = waitLinkBytes; + } + + /* Add a new WAIT/LINK command sequence. When the command buffer which is + currently being scheduled is fully executed by the GPU, the FE will + jump to this WAIT/LINK sequence. */ + gcmkONERROR(gckWLFE_WaitLink( + hardware, + waitLinkLogical, + waitLinkAddress, + offset, + &waitLinkBytes, + &waitOffset, + &waitSize + )); + + if (Command->newQueue) + { + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + 0, + Command->logical, + exitBytes + )); + } + else + { + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + waitLinkLogical, + exitBytes + )); + } + + /* Determine the location of the TAIL in the command buffer. */ + commandBufferTail + = commandBufferLogical + + commandBufferSize + - CommandBuffer->reservedTail; + + /* Generate command which writes out commit stamp. */ + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT)) + { + gctUINT32 bytes; + + gcmkONERROR(gckHARDWARE_Fence( + hardware, + gcvENGINE_RENDER, + commandBufferTail, + Command->fence->address, + Command->commitStamp, + &bytes + )); + + commandBufferTail += bytes; + } + + /* Generate a LINK from the end of the command buffer being scheduled + back to the kernel command queue. */ +#if !gcdSECURITY + if (Shared == gcvFALSE) + { + gcmkONERROR(gckWLFE_Link( + hardware, + commandBufferTail, + exitAddress, + exitBytes, + &linkBytes, + &exitLinkLow, + &exitLinkHigh + )); + } + else + { + gctUINT8_PTR link = commandBufferTail + CommandBuffer->exitIndex * 16; + gctSIZE_T bytes = 8; + + gcmkONERROR(gckWLFE_ChipEnable( + hardware, + link, + (gceCORE_3D_MASK)(1 << hardware->kernel->chipID), + &bytes + )); + + link += bytes; + + gcmkONERROR(gckWLFE_Link( + hardware, + link, + exitAddress, + exitBytes, + &linkBytes, + &exitLinkLow, + &exitLinkHigh + )); + + link += linkBytes; + } +#endif + + gcmkONERROR(gckVIDMEM_HANDLE_Lookup( + Command->kernel, + ProcessID, + CommandBuffer->videoMemNode, + &commandBufferVideoMem + )); + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + commandBufferVideoMem, + CommandBuffer->startOffset, + commandBufferLogical, + commandBufferSize + )); + +#if gcdRECORD_COMMAND + gckRECORDER_Record( + Command->recorder, + commandBufferLogical + offset, + commandBufferSize - offset, + gcvNULL, + 0xFFFFFFFF + ); + + gckRECORDER_AdvanceIndex(Command->recorder, Command->commitStamp); +#endif + +#if gcdSECURITY + /* Submit command buffer to trust zone. */ + gckKERNEL_SecurityExecute( + Command->kernel, + commandBufferLogical + offset, + commandBufferSize - offset - 8 + ); +#else +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* + * Skip link to entryAddress. + * Instead, we directly link to final wait link position. + */ + gcmkONERROR(gckWLFE_Link( + hardware, + Command->waitPos.logical, + waitLinkAddress, + waitLinkBytes, + &Command->waitPos.size, + &entryLinkLow, + &entryLinkHigh + )); +# else + /* Generate a LINK from the previous WAIT/LINK command sequence to the + entry determined above (either the context or the command buffer). + This LINK replaces the WAIT instruction from the previous WAIT/LINK + pair, therefore we use WAIT metrics for generation of this LINK. + This action will execute the entire sequence. */ + gcmkONERROR(gckWLFE_Link( + hardware, + Command->waitPos.logical, + entryAddress, + entryBytes, + &Command->waitPos.size, + &entryLinkLow, + &entryLinkHigh + )); +# endif +#endif + +#if gcdLINK_QUEUE_SIZE + /* TODO: What's it? */ + if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_USER_COMMAND) + { + gcuQUEUEDATA data; + + gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); + + data.linkData.start = entryAddress; + data.linkData.end = entryAddress + entryBytes; + data.linkData.linkLow = entryLinkLow; + data.linkData.linkHigh = entryLinkHigh; + + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + + if (commandBufferAddress + offset != entryAddress) + { + data.linkData.start = commandBufferAddress + offset; + data.linkData.end = commandBufferAddress + commandBufferSize; + data.linkData.linkLow = commandLinkLow; + data.linkData.linkHigh = commandLinkHigh; + + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + } + + if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) + { + data.linkData.start = exitAddress; + data.linkData.end = exitAddress + exitBytes; + data.linkData.linkLow = exitLinkLow; + data.linkData.linkHigh = exitLinkHigh; + + /* Dump kernel command.*/ + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + } + } +#endif + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->waitPos.videoMem, + Command->waitPos.offset, + Command->waitPos.logical, + Command->waitPos.size + )); + + if (entryAddress != commandBufferAddress + offset) + { + gcmkDUMP(Command->os, "#[context]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_CONTEXT, + contextDumpLogical, + entryAddress, + entryBytes + ); + + /* execute context. */ + gcmkDUMP(Command->os, + "@[execute 0 0 0x%08X 0x%08X]", + entryAddress + offset, + entryBytes - offset - 8 + ); + } + + gcmkDUMP(Command->os, "#[command: user]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_COMMAND, + commandBufferLogical + offset, + commandBufferAddress + offset, + commandBufferSize - offset + ); + + /* execute user commands. */ + gcmkDUMP( + Command->os, + "@[execute 0 0 0x%08X 0x%08X]", + commandBufferAddress + + CommandBuffer->reservedHead, + commandBufferSize + - CommandBuffer->reservedHead + - CommandBuffer->reservedTail + ); + + gcmkDUMP(Command->os, "#[wait-link]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + waitLinkLogical, + waitLinkAddress, + waitLinkBytes + ); + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + gcmkDUMP( + Command->os, + "#[null driver: below command skipped link to 0x%08X 0x%08X]", + entryAddress, + entryBytes + ); +#endif + + gcmkDUMP(Command->os, "#[link: break prev wait-link]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + Command->waitPos.logical, + Command->waitPos.address, + Command->waitPos.size + ); + + /* Update the current pipe. */ + Command->pipeSelect = CommandBuffer->exitPipe; + + /* Update command queue offset. */ + Command->offset += waitLinkBytes; + Command->newQueue = gcvFALSE; + + /* Update address of last WAIT. */ + Command->waitPos.videoMem = Command->videoMem; + Command->waitPos.offset = Command->offset - waitLinkBytes + waitOffset; + Command->waitPos.logical = (gctUINT8_PTR)waitLinkLogical + waitOffset; + Command->waitPos.address = waitLinkAddress + waitOffset; + Command->waitPos.size = waitSize; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail( + hardware, Command->logical, Command->offset + )); + + /* Release the context switching mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + contextAcquired = gcvFALSE; + + /* Release the command queue. */ + gcmkONERROR(gckCOMMAND_ExitCommit(Command, gcvFALSE)); + commitEntered = gcvFALSE; + + if (status == gcvSTATUS_INTERRUPTED) + { + gcmkTRACE( + gcvLEVEL_INFO, + "%s(%d): Intterupted in gckEVENT_Submit", + __FUNCTION__, __LINE__ + ); + status = gcvSTATUS_OK; + } + else + { + gcmkONERROR(status); + } + +#ifdef __QNXNTO__ + if(userCommandBufferLogicalMapped) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + Command->kernel, + commandBufferVideoMem, + ProcessID, + gcvFALSE, + gcvFALSE + )); + + userCommandBufferLogicalMapped =gcvFALSE; + } +#endif + + /* Return status. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (contextAcquired) + { + /* Release the context switching mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + } + + if (commitEntered) + { + /* Release the command queue mutex. */ + gcmkVERIFY_OK(gckCOMMAND_ExitCommit(Command, gcvFALSE)); + } + +#ifdef __QNXNTO__ + if (userCommandBufferLogicalMapped) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userCommandBufferLogical, + 0, + commandBufferLogical)); + } +#endif + + /* Return status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_CommitAsyncOnce( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer + ) +{ + gceSTATUS status; + gckHARDWARE hardware = Command->kernel->hardware; + gctBOOL available = gcvFALSE; + gctBOOL acquired = gcvFALSE; + gctUINT8_PTR commandBufferLogical; + gctUINT8_PTR commandBufferTail; + gctUINT commandBufferSize; + gctUINT32 commandBufferAddress; + gctUINT32 fenceBytes; + gctUINT32 oldValue; + gctUINT32 flushBytes; + gcsPATCH_LIST_VARIABLE patchListVar = {0, 0}; + + gcmkHEADER(); + + gcmkVERIFY_OK( + _HandlePatchList(Command, CommandBuffer, &patchListVar)); + + gckOS_AtomicExchange(Command->os, + hardware->pageTableDirty[gcvENGINE_BLT], + 0, + &oldValue); + + if (oldValue) + { + gckHARDWARE_FlushAsyncMMU(hardware, gcvNULL, &flushBytes); + + gcmkASSERT(flushBytes <= CommandBuffer->reservedHead); + + /* Compute the command buffer entry to insert the flushMMU commands. */ + commandBufferLogical = (gctUINT8_PTR)gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset + + CommandBuffer->reservedHead + - flushBytes; + + commandBufferAddress = CommandBuffer->address + + CommandBuffer->startOffset + + CommandBuffer->reservedHead + - flushBytes; + + gckHARDWARE_FlushAsyncMMU(hardware, commandBufferLogical, &flushBytes); + } + else + { + /* Compute the command buffer entry. */ + commandBufferLogical = (gctUINT8_PTR)gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset + + CommandBuffer->reservedHead; + + commandBufferAddress = CommandBuffer->address + + CommandBuffer->startOffset + + CommandBuffer->reservedHead; + + flushBytes = 0; + } + + commandBufferTail = (gctUINT8_PTR)gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset + + CommandBuffer->size + - CommandBuffer->reservedTail; + + gcmkONERROR(gckHARDWARE_Fence( + hardware, + gcvENGINE_BLT, + commandBufferTail, + Command->fence->address, + Command->commitStamp, + &fenceBytes + )); + + gcmkASSERT(fenceBytes <= CommandBuffer->reservedTail); + + commandBufferSize = CommandBuffer->size + - CommandBuffer->reservedHead + - CommandBuffer->reservedTail + + flushBytes + + fenceBytes; + + gckOS_AcquireMutex(Command->os, Command->mutexContext, gcvINFINITE); + acquired = gcvTRUE; + + /* Acquire a slot. */ + for (;;) + { + gcmkONERROR(gckASYNC_FE_ReserveSlot(hardware, &available)); + + if (available) + { + break; + } + else + { + gcmkTRACE_ZONE(gcvLEVEL_INFO, _GC_OBJ_ZONE, "No available slot, have to wait"); + + gckOS_Delay(Command->os, 1); + } + } + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* Skip submit to hardware for NULL driver. */ + gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); +#endif + + gcmkDUMP(Command->os, "#[async-command: user]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_ASYNC_COMMAND, + commandBufferLogical, + commandBufferAddress, + commandBufferSize + ); + + gcmkDUMP( + Command->os, + "@[execute 1 0 0x%08X 0x%08X]", + commandBufferAddress + + CommandBuffer->reservedHead, + CommandBuffer->size + - CommandBuffer->reservedHead + - CommandBuffer->reservedTail + ); + +#if !gcdNULL_DRIVER + /* Execute command buffer. */ + gckASYNC_FE_Execute(hardware, commandBufferAddress, commandBufferSize); +#endif + + gckOS_ReleaseMutex(Command->os, Command->mutexContext); + acquired = gcvFALSE; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gckOS_ReleaseMutex(Command->os, Command->mutexContext); + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_CommitMultiChannelOnce( + IN gckCOMMAND Command, + IN gckCONTEXT Context, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gctUINT8_PTR commandBufferLogical; + gctUINT commandBufferSize; + gctUINT32 commandBufferAddress; + gckHARDWARE hardware; + gctUINT64 bit; + gcsPATCH_LIST_VARIABLE patchListVar = {0, 0}; + + gcmkHEADER_ARG("priority=%d channelId=%d videoMemNode=%u size=0x%x patchHead=%p", + CommandBuffer->priority, CommandBuffer->channelId, + CommandBuffer->videoMemNode, CommandBuffer->size, + gcmUINT64_TO_PTR(CommandBuffer->patchHead)); + + gcmkASSERT(Command->feType == gcvHW_FE_MULTI_CHANNEL); + + hardware = Command->kernel->hardware; + + gcmkVERIFY_OK( + _HandlePatchList(Command, CommandBuffer, &patchListVar)); + + /* Check flush mcfe MMU cache. */ + gcmkONERROR(_CheckFlushMcfeMMU(Command, hardware)); + + /* Compute the command buffer entry and the size. */ + commandBufferLogical + = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset + + CommandBuffer->reservedHead; + + commandBufferAddress = CommandBuffer->address + + CommandBuffer->startOffset + + CommandBuffer->reservedHead; + + /* reservedTail bytes are not used, because fence not enable. */ + commandBufferSize + = CommandBuffer->size + - CommandBuffer->reservedHead + - CommandBuffer->reservedTail; + + if (commandBufferSize & 8) + { + /* + * Need 16 byte alignment for MCFE command size. + * command is already 8 byte aligned, if not 16 byte aligned, + * we need append 8 bytes. + */ + gctUINT32 nop[2]; + gctSIZE_T bytes = 8; + gctUINT8_PTR tail = commandBufferLogical + commandBufferSize; + + gckMCFE_Nop(hardware, nop, &bytes); + gcmkASSERT(bytes == 8); + + gckOS_WriteMemory(Command->os, tail, nop[0]); + gckOS_WriteMemory(Command->os, tail + 4, nop[1]); + + commandBufferSize += 8; + } + + /* Large command buffer size does not make sense. */ + gcmkASSERT(commandBufferSize < 0x800000); + + + if (CommandBuffer->channelId != 0) + { + /* Sync from the system channel. */ + gcmkONERROR(_SyncFromSystemChannel( + Command, + (gctBOOL)CommandBuffer->priority, + (gctUINT32)CommandBuffer->channelId + )); + } + + Command->currContext = Context; + + gckOS_AcquireMutex(Command->os, Command->mutexQueue, gcvINFINITE); + acquired = gcvTRUE; + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* Skip submit to hardware for NULL driver. */ + gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); +#endif + + gcmkDUMP(Command->os, "#[mcfe-command: user]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_COMMAND, + commandBufferLogical, + commandBufferAddress, + commandBufferSize + ); + + gcmkDUMP(Command->os, + "@[execute %d %d 0x%08X 0x%08X]", + CommandBuffer->channelId, + CommandBuffer->priority, + commandBufferAddress, + commandBufferSize); + +#if !gcdNULL_DRIVER + /* Execute command buffer. */ + gcmkONERROR(gckMCFE_Execute( + hardware, + (gctBOOL)CommandBuffer->priority, + (gctUINT32)CommandBuffer->channelId, + commandBufferAddress, + commandBufferSize + )); +#endif + + bit = 1ull << CommandBuffer->channelId; + + /* This channel is dirty. */ + Command->dirtyChannel[CommandBuffer->priority ? 1 : 0] |= bit; + + gckOS_ReleaseMutex(Command->os, Command->mutexQueue); + acquired = gcvFALSE; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gckOS_ReleaseMutex(Command->os, Command->mutexQueue); + } + + gcmkFOOTER(); + return status; +} + + +/******************************************************************************* +** +** gckCOMMAND_Commit +** +** Commit command buffers to the command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** gcsHAL_SUBCOMMIT * SubCommit +** Commit information, includes context, delta, and command buffer +* locations. +** +** gctUINT32 ProcessID +** Current process ID. +** +** OUTPUT: +** gctBOOL *contextSwitched +** pass context Switch flag to upper +*/ +gceSTATUS +gckCOMMAND_Commit( + IN gckCOMMAND Command, + IN gcsHAL_SUBCOMMIT * SubCommit, + IN gctUINT32 ProcessId, + IN gctBOOL Shared, + OUT gctUINT64_PTR CommitStamp, + INOUT gctBOOL *contextSwitched + ) +{ + gceSTATUS status; + gcsSTATE_DELTA_PTR delta = gcmUINT64_TO_PTR(SubCommit->delta); + gckCONTEXT context = gcvNULL; + gcsHAL_COMMAND_LOCATION *cmdLoc = &SubCommit->commandBuffer; + gcsHAL_COMMAND_LOCATION _cmdLoc; + gctPOINTER userPtr = gcvNULL; + gctBOOL needCopy = gcvFALSE; + + gcmkHEADER_ARG("Command=%p SubCommit=%p delta=%p context=%u pid=%u", + Command, SubCommit, delta, SubCommit->context, ProcessId); + + gcmkVERIFY_OK(gckOS_QueryNeedCopy(Command->os, ProcessId, &needCopy)); + + if (SubCommit->context) + { + context = gckKERNEL_QueryPointerFromName( + Command->kernel, + (gctUINT32)(SubCommit->context) + ); + } + + do + { + gctUINT64 next; + + /* Skip the first nested one. */ + if (userPtr) + { + /* Copy/map command buffer location from user. */ + if (needCopy) + { + cmdLoc = &_cmdLoc; + + status = gckOS_CopyFromUserData( + Command->os, + cmdLoc, + userPtr, + gcmSIZEOF(gcsHAL_COMMAND_LOCATION) + ); + } + else + { + status = gckOS_MapUserPointer( + Command->os, + userPtr, + gcmSIZEOF(gcsHAL_COMMAND_LOCATION), + (gctPOINTER *)&cmdLoc + ); + } + + if (gcmIS_ERROR(status)) + { + userPtr = gcvNULL; + + gcmkONERROR(status); + } + } + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + /* Commit command buffers. */ + status = _CommitWaitLinkOnce(Command, + context, + cmdLoc, + delta, + ProcessId, + Shared, + contextSwitched); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + status = _CommitMultiChannelOnce(Command, context, cmdLoc); + } + else + { + gcmkASSERT(Command->feType == gcvHW_FE_ASYNC); + + status = _CommitAsyncOnce(Command, cmdLoc); + } + + if (status != gcvSTATUS_INTERRUPTED) + { + gcmkONERROR(status); + } + + /* Do not need context or delta for later commands. */ + context = gcvNULL; + delta = gcvNULL; + + next = cmdLoc->next; + + /* Unmap user pointer if mapped. */ + if (!needCopy && userPtr) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + gcmSIZEOF(gcsHAL_COMMAND_LOCATION), + cmdLoc + )); + } + + /* Advance to next command buffer location from user. */ + userPtr = gcmUINT64_TO_PTR(next); + } + while (userPtr); + + if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + /* + * Semphore synchronization. + * + * Here we blindly sync dirty other channels to the system channel. + * The scenario to sync channels to the system channel: + * 1. Need to sync channels who sent semaphores. + * 2. Need to sync dirty channels when event(interrupt) is to sent. + * 3. Need to sync dirty channels when system channel need run something + * such as flush mmu. + * + * When power management is on, blindly sync dirty channels is OK because + * there's always a event(intrrupt). + * + * The only condition we sync more than needed is: + * a. power manangement is off. + * b. no user event is attached when commit. + * c. no user event is to be submitted in next ioctl. + * That's a rare condition. + * + * Conclusion is that, blindly sync dirty channels is a good choice for + * now. + */ + gcmkONERROR(_SyncToSystemChannel(Command, Command->dirtyChannel)); + } + + /* Output commit stamp. */ + *CommitStamp = Command->commitStamp; + Command->commitStamp++; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (!needCopy && userPtr) + { + gckOS_UnmapUserPointer( + Command->os, + userPtr, + gcmSIZEOF(gcsHAL_COMMAND_LOCATION), + cmdLoc + ); + } + + gcmkFOOTER(); + return status; +} + + +/******************************************************************************* +** +** gckCOMMAND_Reserve +** +** Reserve space in the command queue. Also acquire the command queue mutex. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctSIZE_T RequestedBytes +** Number of bytes previously reserved. +** +** OUTPUT: +** +** gctPOINTER * Buffer +** Pointer to a variable that will receive the address of the reserved +** space. +** +** gctSIZE_T * BufferSize +** Pointer to a variable that will receive the number of bytes +** available in the command queue. +*/ +gceSTATUS +gckCOMMAND_Reserve( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes, + OUT gctPOINTER * Buffer, + OUT gctUINT32 * BufferSize + ) +{ + gceSTATUS status; + gctUINT32 bytes; + gctUINT32 requiredBytes; + gctUINT32 requestedAligned; + + gcmkHEADER_ARG("Command=%p RequestedBytes=%lu", Command, RequestedBytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + /* Compute aligned number of reuested bytes. */ + requestedAligned = gcmALIGN(RequestedBytes, Command->alignment); + + /* Another WAIT/LINK command sequence will have to be appended after + the requested area being reserved. Compute the number of bytes + required for WAIT/LINK at the location after the reserved area. */ + gcmkONERROR(gckWLFE_WaitLink( + Command->kernel->hardware, + gcvNULL, + ~0U, + Command->offset + requestedAligned, + &requiredBytes, + gcvNULL, + gcvNULL + )); + + /* Compute total number of bytes required. */ + requiredBytes += requestedAligned; + } + else + { + requiredBytes = gcmALIGN(RequestedBytes, 8); + } + + /* Compute number of bytes available in command queue. */ + bytes = Command->pageSize - Command->offset; + + /* Is there enough space in the current command queue? */ + if (bytes < requiredBytes) + { + /* Create a new command queue. */ + gcmkONERROR(_NewQueue(Command, gcvFALSE)); + + /* Recompute the number of bytes in the new kernel command queue. */ + bytes = Command->pageSize - Command->offset; + + /* Still not enough space? */ + if (bytes < requiredBytes) + { + /* Rare case, not enough room in command queue. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + } + + /* Return pointer to empty slot command queue. */ + *Buffer = (gctUINT8 *) Command->logical + Command->offset; + + /* Return number of bytes left in command queue. */ + *BufferSize = bytes; + + /* Success. */ + gcmkFOOTER_ARG("*Buffer=0x%x *BufferSize=%lu", *Buffer, *BufferSize); + return gcvSTATUS_OK; + +OnError: + /* Return status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Execute +** +** Execute a previously reserved command queue by appending a WAIT/LINK command +** sequence after it and modifying the last WAIT into a LINK command. The +** command FIFO mutex will be released whether this function succeeds or not. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctSIZE_T RequestedBytes +** Number of bytes previously reserved. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Execute( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes + ) +{ + gceSTATUS status; + + gctUINT8_PTR waitLinkLogical; + gctUINT32 waitLinkAddress; + gctUINT32 waitLinkOffset; + gctUINT32 waitLinkBytes; + + gctUINT32 waitOffset; + gctUINT32 waitBytes; + + gctUINT32 linkLow, linkHigh; + + gctPOINTER execLogical; + gctUINT32 execAddress; + gctUINT32 execBytes; + + gcmkHEADER_ARG("Command=%p RequestedBytes=%lu", Command, RequestedBytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Compute offset for WAIT/LINK. */ + waitLinkOffset = Command->offset + RequestedBytes; + + /* Compute number of bytes left in command queue. */ + waitLinkBytes = Command->pageSize - waitLinkOffset; + + /* Compute the location if WAIT/LINK command sequence. */ + waitLinkLogical = (gctUINT8_PTR) Command->logical + waitLinkOffset; + waitLinkAddress = Command->address + waitLinkOffset; + + /* Append WAIT/LINK in command queue. */ + gcmkONERROR(gckWLFE_WaitLink( + Command->kernel->hardware, + waitLinkLogical, + waitLinkAddress, + waitLinkOffset, + &waitLinkBytes, + &waitOffset, + &waitBytes + )); + + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ + execLogical = Command->logical; + execAddress = Command->address; + execBytes = Command->offset + RequestedBytes + waitLinkBytes; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + 0, + execLogical, + execBytes + )); + } + else + { + /* Still within the preexisting command queue, jump directly to the + reserved area. */ + execLogical = (gctUINT8 *) Command->logical + Command->offset; + execAddress = Command->address + Command->offset; + execBytes = RequestedBytes + waitLinkBytes; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + execLogical, + execBytes + )); + } + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* + * Skip link to execAddress. + * Instead, we directly link to final wait link position. + */ + gcmkONERROR(gckWLFE_Link( + Command->kernel->hardware, + Command->waitPos.logical, + waitLinkAddress, + waitLinkBytes, + &Command->waitPos.size, + &linkLow, + &linkHigh + )); +#else + /* Convert the last WAIT into a LINK. */ + gcmkONERROR(gckWLFE_Link( + Command->kernel->hardware, + Command->waitPos.logical, + execAddress, + execBytes, + &Command->waitPos.size, + &linkLow, + &linkHigh + )); +#endif + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->waitPos.videoMem, + Command->waitPos.offset, + Command->waitPos.logical, + Command->waitPos.size + )); + +#if gcdLINK_QUEUE_SIZE + if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) + { + gcuQUEUEDATA data; + + gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); + + data.linkData.start = execAddress; + data.linkData.end = execAddress + execBytes; + data.linkData.linkLow = linkLow; + data.linkData.linkHigh = linkHigh; + + gckQUEUE_Enqueue(&Command->kernel->hardware->linkQueue, &data); + } +#endif + + gcmkDUMP(Command->os, "#[command: kernel execute]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + execLogical, + execAddress, + execBytes + ); + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + gcmkDUMP(Command->os, + "#[null driver: below command skipped link to 0x%08X 0x%08X]", + execAddress, + execBytes); +#endif + + gcmkDUMP(Command->os, "#[link: break prev wait-link]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + Command->waitPos.logical, + Command->waitPos.address, + Command->waitPos.size + ); + + /* Update the pointer to the last WAIT. */ + Command->waitPos.videoMem = Command->videoMem; + Command->waitPos.offset = waitLinkOffset + waitOffset; + Command->waitPos.logical = (gctUINT8_PTR)waitLinkLogical + waitOffset; + Command->waitPos.address = waitLinkAddress + waitOffset; + Command->waitPos.size = waitBytes; + + /* Update the command queue. */ + Command->offset += RequestedBytes + waitLinkBytes; + Command->newQueue = gcvFALSE; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail( + Command->kernel->hardware, Command->logical, Command->offset + )); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckCOMMAND_ExecuteAsync( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctBOOL available; + gctPOINTER execLogical; + gctUINT32 execAddress; + gctUINT32 execBytes; + + hardware = Command->kernel->hardware; + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ + execLogical = Command->logical; + execAddress = Command->address; + execBytes = Command->offset + RequestedBytes; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + 0, + execLogical, + execBytes + )); + } + else + { + /* Still within the preexisting command queue, jump directly to the + reserved area. */ + execLogical = (gctUINT8 *) Command->logical + Command->offset; + execAddress = Command->address + Command->offset; + execBytes = RequestedBytes; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + execLogical, + execBytes + )); + } + + /* Acquire a slot. */ + for (;;) + { + gcmkONERROR(gckASYNC_FE_ReserveSlot(hardware, &available)); + + if (available) + { + break; + } + else + { + gckOS_Delay(Command->os, 1); + } + } + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* Skip submit to hardware for NULL driver. */ + gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); +#endif + + gcmkDUMP(Command->os, "#[async-command: kernel execute]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + execLogical, + execAddress, + execBytes + ); + +#if !gcdNULL_DRIVER + /* Send descriptor. */ + gckASYNC_FE_Execute(hardware, execAddress, execBytes); +#endif + + /* Update the command queue. */ + Command->offset += RequestedBytes; + Command->newQueue = gcvFALSE; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +gceSTATUS +gckCOMMAND_ExecuteMultiChannel( + IN gckCOMMAND Command, + IN gctBOOL Priority, + IN gctUINT32 ChannelId, + IN gctUINT32 RequestedBytes + ) +{ + gceSTATUS status; + gctPOINTER execLogical; + gctUINT32 execAddress; + gctUINT32 execBytes; + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ + execLogical = Command->logical; + execAddress = Command->address; + execBytes = Command->offset + RequestedBytes; + } + else + { + /* Still within the preexisting command queue, jump directly to the + reserved area. */ + execLogical = (gctUINT8 *) Command->logical + Command->offset; + execAddress = Command->address + Command->offset; + execBytes = RequestedBytes; + } + + if (execBytes & 8) + { + gctSIZE_T bytes = 8; + gctUINT8_PTR tail = (gctUINT8_PTR)execLogical + execBytes; + + gckMCFE_Nop(Command->kernel->hardware, tail, &bytes); + gcmkASSERT(bytes == 8); + + execBytes += 8; + RequestedBytes += 8; + } + + if (Command->newQueue) + { + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + 0, + execLogical, + execBytes + )); + } + else + { + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + execLogical, + execBytes + )); + } + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* Skip submit to hardware for NULL driver. */ + gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); +#endif + + gcmkDUMP(Command->os, "#[mcfe-command: kernel execute]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + execLogical, + execAddress, + execBytes + ); + + gcmkDUMP(Command->os, + "@[execute %u %u 0x%08X 0x%08X]", + ChannelId, Priority, execAddress, execBytes); + +#if !gcdNULL_DRIVER + /* Send descriptor. */ + gcmkONERROR( + gckMCFE_Execute(Command->kernel->hardware, + Priority, + ChannelId, + execAddress, + execBytes)); +#endif + + /* Update the command queue. */ + Command->offset += RequestedBytes; + Command->newQueue = gcvFALSE; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Stall +** +** The calling thread will be suspended until the command queue has been +** completed. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Stall( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ) +{ + gckOS os; + gckHARDWARE hardware; + gckEVENT eventObject; + gceSTATUS status; + gctSIGNAL signal = gcvNULL; + gctUINT timer = 0; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Extract the gckOS object pointer. */ + os = Command->os; + gcmkVERIFY_OBJECT(os, gcvOBJ_OS); + + /* Extract the gckHARDWARE object pointer. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Extract the gckEVENT object pointer. */ + eventObject = Command->kernel->eventObj; + gcmkVERIFY_OBJECT(eventObject, gcvOBJ_EVENT); + + /* Allocate the signal. */ + gcmkONERROR(gckOS_CreateSignal(os, gcvTRUE, &signal)); + + /* Append the EVENT command to trigger the signal. */ + gcmkONERROR(gckEVENT_Signal(eventObject, signal, gcvKERNEL_PIXEL)); + + /* Submit the event queue. */ + gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower)); + + gcmkDUMP(Command->os, "#[kernel.stall]"); + + if (status == gcvSTATUS_CHIP_NOT_READY) + { + /* Error. */ + goto OnError; + } + + do + { + /* Wait for the signal. */ + status = gckOS_WaitSignal(os, signal, !FromPower, gcdGPU_ADVANCETIMER); + + if (status == gcvSTATUS_TIMEOUT) + { +#if gcmIS_DEBUG(gcdDEBUG_CODE) + gctUINT32 idle; + + /* Read idle register. */ + gcmkVERIFY_OK(gckHARDWARE_GetIdle( + hardware, gcvFALSE, &idle + )); + + gcmkTRACE( + gcvLEVEL_ERROR, + "%s(%d): idle=%08x", + __FUNCTION__, __LINE__, idle + ); + + gcmkVERIFY_OK(gckOS_MemoryBarrier(os, gcvNULL)); +#endif + + /* Advance timer. */ + timer += gcdGPU_ADVANCETIMER; + } + else if (status == gcvSTATUS_INTERRUPTED) + { + gcmkONERROR(gcvSTATUS_INTERRUPTED); + } + + } + while (gcmIS_ERROR(status)); + + /* Bail out on timeout. */ + if (gcmIS_ERROR(status)) + { + /* Broadcast the stuck GPU. */ + gcmkONERROR(gckOS_Broadcast( + os, hardware, gcvBROADCAST_GPU_STUCK + )); + } + + /* Delete the signal. */ + gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (signal != gcvNULL) + { + /* Free the signal. */ + gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +#if (gcdENABLE_3D) +static gceSTATUS +_AttachWaitLinkFECommand( + IN gckCOMMAND Command, + OUT gckCONTEXT * Context, + OUT gctSIZE_T * MaxState, + OUT gctUINT32 * NumStates, + IN gctUINT32 ProcessID + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Acquire the context switching mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, Command->mutexContext, gcvINFINITE + )); + acquired = gcvTRUE; + + /* Construct a gckCONTEXT object. */ + gcmkONERROR(gckCONTEXT_Construct( + Command->os, + Command->kernel->hardware, + ProcessID, + Context + )); + + /* Return the number of states in the context. */ + * MaxState = (* Context)->maxState; + * NumStates = (* Context)->numStates; + + /* Release the context switching mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + + /* Success. */ + gcmkFOOTER_ARG("*Context=0x%x", *Context); + return gcvSTATUS_OK; + +OnError: + /* Release mutex. */ + if (acquired) + { + /* Release the context switching mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Attach +** +** Attach user process. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** gctUINT32 ProcessID +** Current process ID. +** +** OUTPUT: +** +** gckCONTEXT * Context +** Pointer to a variable that will receive a pointer to a new +** gckCONTEXT object. +** +** gctSIZE_T * StateCount +** Pointer to a variable that will receive the number of states +** in the context buffer. +*/ +gceSTATUS +gckCOMMAND_Attach( + IN gckCOMMAND Command, + OUT gckCONTEXT * Context, + OUT gctSIZE_T * MaxState, + OUT gctUINT32 * NumStates, + IN gctUINT32 ProcessID + ) +{ + gctUINT32 allocationSize; + gctPOINTER pointer; + gceSTATUS status; + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + status = _AttachWaitLinkFECommand(Command, + Context, + MaxState, + NumStates, + ProcessID); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + /* + * For mcfe, we only allocate context which is used to + * store profile counters. + */ + allocationSize = gcmSIZEOF(struct _gckCONTEXT); + + /* Allocate the object. */ + gckOS_Allocate(Command->os, allocationSize, &pointer); + if (!pointer) + { + return gcvSTATUS_OUT_OF_MEMORY; + } + *Context = pointer; + /* Reset the entire object. */ + gckOS_ZeroMemory(*Context, allocationSize); + + /* Initialize the gckCONTEXT object. */ + (*Context)->object.type = gcvOBJ_CONTEXT; + (*Context)->os = Command->os; + (*Context)->hardware = Command->kernel->hardware; + *MaxState = 0; + *NumStates = 0; + + status = gcvSTATUS_OK; + } + else + { + /* Nothing to do. */ + *Context = gcvNULL; + *MaxState = 0; + *NumStates = 0; + + status = gcvSTATUS_OK; + } + + return status; +} +#endif + +static gceSTATUS +_DetachWaitLinkFECommand( + IN gckCOMMAND Command, + IN gckCONTEXT Context + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Command=%p Context=%p", Command, Context); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Acquire the context switching mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, Command->mutexContext, gcvINFINITE + )); + acquired = gcvTRUE; + + /* Construct a gckCONTEXT object. */ + gcmkONERROR(gckCONTEXT_Destroy(Context)); + + if (Command->currContext == Context) + { + /* Detach from gckCOMMAND object if the destoryed context is current context. */ + Command->currContext = gcvNULL; + } + + /* Release the context switching mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + + /* Return the status. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + /* Release mutex. */ + if (acquired) + { + /* Release the context switching mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Detach +** +** Detach user process. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** gckCONTEXT Context +** Pointer to a gckCONTEXT object to be destroyed. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Detach( + IN gckCOMMAND Command, + IN gckCONTEXT Context + ) +{ + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + return _DetachWaitLinkFECommand(Command, Context); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + gcmkOS_SAFE_FREE(Context->os, Context); + return gcvSTATUS_OK; + } + else + { + /* Nothing to do. */ + return gcvSTATUS_OK; + } +} + +static void +_DumpBuffer( + IN gctPOINTER Buffer, + IN gctUINT32 GpuAddress, + IN gctSIZE_T Size + ) +{ + gctSIZE_T i, line, left; + gctUINT32_PTR data = Buffer; + + line = Size / 32; + left = Size % 32; + + for (i = 0; i < line; i++) + { + gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]); + data += 8; + GpuAddress += 8 * 4; + } + + switch(left) + { + case 28: + gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6]); + break; + case 24: + gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5]); + break; + case 20: + gcmkPRINT("%08X : %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4]); + break; + case 16: + gcmkPRINT("%08X : %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3]); + break; + case 12: + gcmkPRINT("%08X : %08X %08X %08X", + GpuAddress, data[0], data[1], data[2]); + break; + case 8: + gcmkPRINT("%08X : %08X %08X", + GpuAddress, data[0], data[1]); + break; + case 4: + gcmkPRINT("%08X : %08X", + GpuAddress, data[0]); + break; + default: + break; + } +} + +/******************************************************************************* +** +** gckCOMMAND_DumpExecutingBuffer +** +** Dump the command buffer which GPU is executing. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_DumpExecutingBuffer( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + gctUINT32 gpuAddress; + gctPOINTER entry = gcvNULL; + gckOS os = Command->os; + gckKERNEL kernel = Command->kernel; + gctUINT32 i; + gckQUEUE queue = &kernel->hardware->linkQueue; + gctSIZE_T bytes; + gctUINT32 offset; + gctPOINTER entryDump; + gctUINT8 processName[24] = {0}; + + gcmkPRINT("**************************\n"); + gcmkPRINT("**** COMMAND BUF DUMP ****\n"); + gcmkPRINT("**************************\n"); + + gcmkPRINT(" Submitted commit stamp = %lld", Command->commitStamp - 1); + gcmkPRINT(" Executed commit stamp = %lld", *(gctUINT64_PTR)Command->fence->logical); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); + gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); + + gcmkPRINT("DMA Address 0x%08X", gpuAddress); + + /* Find GPU address in video memory list. */ + status = gckVIDMEM_NODE_Find(kernel, gpuAddress, &nodeObject, &offset); + + if (gcmIS_SUCCESS(status) && nodeObject->type == gcvVIDMEM_TYPE_COMMAND) + { + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + kernel, + nodeObject, + gcvFALSE, + gcvFALSE, + &entryDump + )); + + gcmkVERIFY_OK(gckVIDMEM_NODE_GetSize( + kernel, + nodeObject, + &bytes + )); + + gcmkPRINT("Command buffer around 0x%08X:", gpuAddress); + + /* Align to 4096. */ + offset &= 0xfffff000; + gpuAddress &= 0xfffff000; + + /* Dump max 4096 bytes. */ + bytes = (bytes - offset) > 4096 ? 4096 : (bytes - offset); + + /* Kernel address of page where stall point stay. */ + entryDump = (gctUINT8_PTR)entryDump + offset; + + _DumpBuffer(entryDump, gpuAddress, bytes); + + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + kernel, + nodeObject, + 0, + gcvFALSE, + gcvFALSE + )); + } + else + { + gcmkPRINT("Can not find command buffer around 0x%08X.\n", gpuAddress); + } + + /* new line. */ + gcmkPRINT(""); + + gcmkPRINT("Kernel command buffers:"); + + for (i = 0; i < gcdCOMMAND_QUEUES; i++) + { + entry = Command->queues[i].logical; + gpuAddress = Command->queues[i].address; + + gcmkPRINT("command buffer %d at 0x%08X size %u", + i, gpuAddress, Command->pageSize); + _DumpBuffer(entry, gpuAddress, Command->pageSize); + } + + /* new line. */ + gcmkPRINT(""); + + if (queue->count) + { + gcmkPRINT("Dump Level is %d, dump %d valid record in link queue:", + Command->kernel->stuckDump, queue->count); + } + + for (i = 0; i < queue->count; i++) + { + gcuQUEUEDATA * queueData; + gckLINKDATA linkData; + + gckQUEUE_GetData(queue, i, &queueData); + + linkData = &queueData->linkData; + + /* Get gpu address of this command buffer. */ + gpuAddress = linkData->start; + bytes = linkData->end - gpuAddress; + + processName[0] = '\0'; + gckOS_GetProcessNameByPid(linkData->pid, sizeof(processName), processName); + + gcmkPRINT("Link record %d: [%08X - %08X] from command %08X %08X pid %u (%s):", + i, + linkData->start, + linkData->end, + linkData->linkLow, + linkData->linkHigh, + linkData->pid, + processName); + + /* Find GPU address in video memory list. */ + status = gckVIDMEM_NODE_Find(kernel, gpuAddress, &nodeObject, &offset); + + if (gcmIS_SUCCESS(status) && nodeObject->type == gcvVIDMEM_TYPE_COMMAND) + { + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + kernel, + nodeObject, + gcvFALSE, + gcvFALSE, + &entryDump + )); + + /* Kernel address of page where stall point stay. */ + entryDump = (gctUINT8_PTR)entryDump + offset; + + _DumpBuffer(entryDump, gpuAddress, bytes); + + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + kernel, + nodeObject, + 0, + gcvFALSE, + gcvFALSE + )); + } + else + { + gcmkPRINT("Not found"); + } + + /* new line. */ + gcmkPRINT(""); + } + + return gcvSTATUS_OK; + +OnError: + return status; +} +
diff --git a/hal/kernel/gc_hal_kernel_db.c b/hal/kernel/gc_hal_kernel_db.c new file mode 100644 index 0000000..36ffa46 --- /dev/null +++ b/hal/kernel/gc_hal_kernel_db.c
@@ -0,0 +1,1662 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" + +#define _GC_OBJ_ZONE gcvZONE_DATABASE + +/******************************************************************************* +***** Private fuctions ********************************************************/ + +#define _GetSlot(database, x) \ + (gctUINT32)(gcmPTR_TO_UINT64(x) % gcmCOUNTOF(database->list)) + +/******************************************************************************* +** gckKERNEL_FindDatabase +** +** Find a database identified by a process ID and move it to the head of the +** hash list. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gctUINT32 ProcessID +** ProcessID that identifies the database. +** +** gctBOOL LastProcessID +** gcvTRUE if searching for the last known process ID. gcvFALSE if +** we need to search for the process ID specified by the ProcessID +** argument. +** +** OUTPUT: +** +** gcsDATABASE_PTR * Database +** Pointer to a variable receiving the database structure pointer on +** success. +*/ +gceSTATUS +gckKERNEL_FindDatabase( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctBOOL LastProcessID, + OUT gcsDATABASE_PTR * Database + ) +{ + gceSTATUS status; + gcsDATABASE_PTR database, previous; + gctSIZE_T slot; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d LastProcessID=%d", + Kernel, ProcessID, LastProcessID); + + /* Compute the hash for the database. */ + slot = ProcessID % gcmCOUNTOF(Kernel->db->db); + + /* Acquire the database mutex. */ + gcmkONERROR( + gckOS_AcquireMutex(Kernel->os, Kernel->db->dbMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Check whether we are getting the last known database. */ + if (LastProcessID) + { + /* Use last database. */ + database = Kernel->db->lastDatabase; + + if (database == gcvNULL) + { + /* Database not found. */ + gcmkONERROR(gcvSTATUS_INVALID_DATA); + } + } + else + { + /* Walk the hash list. */ + for (previous = gcvNULL, database = Kernel->db->db[slot]; + database != gcvNULL; + database = database->next) + { + if (database->processID == ProcessID) + { + /* Found it! */ + break; + } + + previous = database; + } + + if (database == gcvNULL) + { + /* Database not found. */ + gcmkONERROR(gcvSTATUS_INVALID_DATA); + } + + if (previous != gcvNULL) + { + /* Move database to the head of the hash list. */ + previous->next = database->next; + database->next = Kernel->db->db[slot]; + Kernel->db->db[slot] = database; + } + } + + /* Release the database mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + + /* Return the database. */ + *Database = database; + + /* Success. */ + gcmkFOOTER_ARG("*Database=0x%x", *Database); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the database mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_DeinitDatabase +** +** De-init a database structure. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gcsDATABASE_PTR Database +** Pointer to the database structure to deinit. +** +** OUTPUT: +** +** Nothing. +*/ +static gceSTATUS +gckKERNEL_DeinitDatabase( + IN gckKERNEL Kernel, + IN gcsDATABASE_PTR Database + ) +{ + gcmkHEADER_ARG("Kernel=%p Database=%p", Kernel, Database); + + if (Database) + { + Database->deleted = gcvFALSE; + + /* Destory handle db. */ + if (Database->refs) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, Database->refs)); + Database->refs = gcvNULL; + } + + if (Database->handleDatabase) + { + gcmkVERIFY_OK(gckKERNEL_DestroyIntegerDatabase(Kernel, Database->handleDatabase)); + Database->handleDatabase = gcvNULL; + } + + if (Database->handleDatabaseMutex) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Database->handleDatabaseMutex)); + Database->handleDatabaseMutex = gcvNULL; + } + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** gckKERNEL_NewRecord +** +** Create a new database record structure and insert it to the head of the +** database. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gcsDATABASE_PTR Database +** Pointer to a database structure. +** +** OUTPUT: +** +** gcsDATABASE_RECORD_PTR * Record +** Pointer to a variable receiving the database record structure +** pointer on success. +*/ +static gceSTATUS +gckKERNEL_NewRecord( + IN gckKERNEL Kernel, + IN gcsDATABASE_PTR Database, + IN gctUINT32 Slot, + OUT gcsDATABASE_RECORD_PTR * Record + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gcsDATABASE_RECORD_PTR record = gcvNULL; + + gcmkHEADER_ARG("Kernel=%p Database=%p", Kernel, Database); + + /* Acquire the database mutex. */ + gcmkONERROR( + gckOS_AcquireMutex(Kernel->os, Kernel->db->dbMutex, gcvINFINITE)); + acquired = gcvTRUE; + + if (Kernel->db->freeRecord != gcvNULL) + { + /* Allocate the record from the free list. */ + record = Kernel->db->freeRecord; + Kernel->db->freeRecord = record->next; + } + else + { + gctPOINTER pointer = gcvNULL; + + /* Allocate the record from the heap. */ + gcmkONERROR(gckOS_Allocate(Kernel->os, + gcmSIZEOF(gcsDATABASE_RECORD), + &pointer)); + + record = pointer; + } + + /* Insert the record in the database. */ + record->next = Database->list[Slot]; + Database->list[Slot] = record; + + /* Release the database mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + + /* Return the record. */ + *Record = record; + + /* Success. */ + gcmkFOOTER_ARG("*Record=0x%x", *Record); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the database mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + } + if (record != gcvNULL) + { + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, record)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_DeleteRecord +** +** Remove a database record from the database and delete its structure. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gcsDATABASE_PTR Database +** Pointer to a database structure. +** +** gceDATABASE_TYPE Type +** Type of the record to remove. +** +** gctPOINTER Data +** Data of the record to remove. +** +** OUTPUT: +** +** gctSIZE_T_PTR Bytes +** Pointer to a variable that receives the size of the record deleted. +** Can be gcvNULL if the size is not required. +*/ +static gceSTATUS +gckKERNEL_DeleteRecord( + IN gckKERNEL Kernel, + IN gcsDATABASE_PTR Database, + IN gceDATABASE_TYPE Type, + IN gctPOINTER Data, + OUT gctSIZE_T_PTR Bytes OPTIONAL + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gcsDATABASE_RECORD_PTR record, previous; + gctUINT32 slot = _GetSlot(Database, Data); + + gcmkHEADER_ARG("Kernel=%p Database=%p Type=%d Data=%p", + Kernel, Database, Type, Data); + + /* Acquire the database mutex. */ + gcmkONERROR( + gckOS_AcquireMutex(Kernel->os, Kernel->db->dbMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Scan the database for this record. */ + for (record = Database->list[slot], previous = gcvNULL; + record != gcvNULL; + record = record->next + ) + { + if ((record->type == Type) + && (record->data == Data) + ) + { + /* Found it! */ + break; + } + + previous = record; + } + + if (record == gcvNULL) + { + /* Ouch! This record is not found? */ + gcmkONERROR(gcvSTATUS_INVALID_DATA); + } + + if (Bytes != gcvNULL) + { + /* Return size of record. */ + *Bytes = record->bytes; + } + + /* Remove record from database. */ + if (previous == gcvNULL) + { + Database->list[slot] = record->next; + } + else + { + previous->next = record->next; + } + + /* Insert record in free list. */ + record->next = Kernel->db->freeRecord; + Kernel->db->freeRecord = record; + + /* Release the database mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + + /* Success. */ + gcmkFOOTER_ARG("*Bytes=%lu", gcmOPT_VALUE(Bytes)); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the database mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_FindRecord +** +** Find a database record from the database. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gcsDATABASE_PTR Database +** Pointer to a database structure. +** +** gceDATABASE_TYPE Type +** Type of the record to remove. +** +** gctPOINTER Data +** Data of the record to remove. +** +** OUTPUT: +** +** gctSIZE_T_PTR Bytes +** Pointer to a variable that receives the size of the record deleted. +** Can be gcvNULL if the size is not required. +*/ +static gceSTATUS +gckKERNEL_FindRecord( + IN gckKERNEL Kernel, + IN gcsDATABASE_PTR Database, + IN gceDATABASE_TYPE Type, + IN gctPOINTER Data, + OUT gcsDATABASE_RECORD_PTR Record + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gcsDATABASE_RECORD_PTR record; + gctUINT32 slot = _GetSlot(Database, Data); + + gcmkHEADER_ARG("Kernel=%p Database=%p Type=%d Data=%p", + Kernel, Database, Type, Data); + + /* Acquire the database mutex. */ + gcmkONERROR( + gckOS_AcquireMutex(Kernel->os, Kernel->db->dbMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Scan the database for this record. */ + for (record = Database->list[slot]; + record != gcvNULL; + record = record->next + ) + { + if ((record->type == Type) + && (record->data == Data) + ) + { + /* Found it! */ + break; + } + } + + if (record == gcvNULL) + { + /* Ouch! This record is not found? */ + gcmkONERROR(gcvSTATUS_INVALID_DATA); + } + + if (Record != gcvNULL) + { + /* Return information of record. */ + gcmkONERROR( + gckOS_MemCopy(Record, record, sizeof(gcsDATABASE_RECORD))); + } + + /* Release the database mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + + /* Success. */ + gcmkFOOTER_ARG("Record=0x%x", Record); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the database mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +***** Public API **************************************************************/ + +/******************************************************************************* +** gckKERNEL_CreateProcessDB +** +** Create a new process database. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gctUINT32 ProcessID +** Process ID used to identify the database. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_CreateProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gcsDATABASE_PTR database = gcvNULL; + gctPOINTER pointer = gcvNULL; + gctBOOL acquired = gcvFALSE; + gctSIZE_T slot; + gctUINT32 i; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d", Kernel, ProcessID); + + /* Compute the hash for the database. */ + slot = ProcessID % gcmCOUNTOF(Kernel->db->db); + + /* Acquire the database mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->db->dbMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Walk the hash list. */ + for (database = Kernel->db->db[slot]; + database != gcvNULL; + database = database->next) + { + if (database->processID == ProcessID) + { + gctINT32 oldVal = 0; + + if (database->deleted) + { + gcmkFATAL("%s(%d): DB of Process=0x%x cannot be reentered since it was in deletion\n", + __FUNCTION__, __LINE__, ProcessID); + gcmkONERROR(gcvSTATUS_INVALID_REQUEST); + } + + gcmkVERIFY_OK(gckOS_AtomIncrement(Kernel->os, database->refs, &oldVal)); + goto OnExit; + } + } + + if (Kernel->db->freeDatabase) + { + /* Allocate a database from the free list. */ + database = Kernel->db->freeDatabase; + Kernel->db->freeDatabase = database->next; + } + else + { + /* Allocate a new database from the heap. */ + gcmkONERROR(gckOS_Allocate(Kernel->os, + gcmSIZEOF(gcsDATABASE), + &pointer)); + + gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsDATABASE)); + + database = pointer; + + gcmkONERROR(gckOS_CreateMutex(Kernel->os, &database->counterMutex)); + } + + /* Initialize the database. */ + /* Save the hash slot. */ + database->slot = slot; + database->processID = ProcessID; + database->vidMem.bytes = 0; + database->vidMem.maxBytes = 0; + database->vidMem.totalBytes = 0; + database->nonPaged.bytes = 0; + database->nonPaged.maxBytes = 0; + database->nonPaged.totalBytes = 0; + database->mapMemory.bytes = 0; + database->mapMemory.maxBytes = 0; + database->mapMemory.totalBytes = 0; + + for (i = 0; i < gcmCOUNTOF(database->list); i++) + { + database->list[i] = gcvNULL; + } + + for (i = 0; i < gcvVIDMEM_TYPE_COUNT; i++) + { + database->vidMemType[i].bytes = 0; + database->vidMemType[i].maxBytes = 0; + database->vidMemType[i].totalBytes = 0; + } + + for (i = 0; i < gcvPOOL_NUMBER_OF_POOLS; i++) + { + database->vidMemPool[i].bytes = 0; + database->vidMemPool[i].maxBytes = 0; + database->vidMemPool[i].totalBytes = 0; + } + + gcmkASSERT(database->refs == gcvNULL); + gcmkONERROR(gckOS_AtomConstruct(Kernel->os, &database->refs)); + gcmkONERROR(gckOS_AtomSet(Kernel->os, database->refs, 1)); + + gcmkASSERT(database->handleDatabase == gcvNULL); + gcmkONERROR(gckKERNEL_CreateIntegerDatabase(Kernel, 64, &database->handleDatabase)); + + gcmkASSERT(database->handleDatabaseMutex == gcvNULL); + gcmkONERROR(gckOS_CreateMutex(Kernel->os, &database->handleDatabaseMutex)); + + /* Insert the database into the hash. */ + database->next = Kernel->db->db[slot]; + Kernel->db->db[slot] = database; + + /* Reset idle timer. */ + Kernel->db->lastIdle = 0; + +OnError: + if (gcmIS_ERROR(status)) + { + gcmkVERIFY_OK(gckKERNEL_DeinitDatabase(Kernel, database)); + + if (pointer) + { + gcmkOS_SAFE_FREE(Kernel->os, pointer); + } + } + +OnExit: + if (acquired) + { + /* Release the database mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_AddProcessDB +** +** Add a record to a process database. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gctUINT32 ProcessID +** Process ID used to identify the database. +** +** gceDATABASE_TYPE TYPE +** Type of the record to add. +** +** gctPOINTER Pointer +** Data of the record to add. +** +** gctPHYS_ADDR Physical +** Physical address of the record to add. +** +** gctSIZE_T Size +** Size of the record to add. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_AddProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gceDATABASE_TYPE Type, + IN gctPOINTER Pointer, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Size + ) +{ + gceSTATUS status; + gcsDATABASE_PTR database; + gcsDATABASE_RECORD_PTR record = gcvNULL; + gcsDATABASE_COUNTERS * count; + gctUINT32 vidMemType; + gcePOOL vidMemPool; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d Type=%d Pointer=%p " + "Physical=%p Size=%lu", + Kernel, ProcessID, Type, Pointer, Physical, Size); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + + /* Decode type. */ + vidMemType = (Type & gcdDB_VIDEO_MEMORY_TYPE_MASK) >> gcdDB_VIDEO_MEMORY_TYPE_SHIFT; + vidMemPool = (Type & gcdDB_VIDEO_MEMORY_POOL_MASK) >> gcdDB_VIDEO_MEMORY_POOL_SHIFT; + + Type &= gcdDATABASE_TYPE_MASK; + + /* Special case the idle record. */ + if (Type == gcvDB_IDLE) + { + gctUINT64 time; + + /* Get the current profile time. */ + gcmkONERROR(gckOS_GetProfileTick(&time)); + + if ((ProcessID == 0) && (Kernel->db->lastIdle != 0)) + { + /* Out of idle, adjust time it was idle. */ + Kernel->db->idleTime += time - Kernel->db->lastIdle; + Kernel->db->lastIdle = 0; + } + else if (ProcessID == 1) + { + /* Save current idle time. */ + Kernel->db->lastIdle = time; + } + +#if gcdDYNAMIC_SPEED + { + /* Test for first call. */ + if (Kernel->db->lastSlowdown == 0) + { + /* Save milliseconds. */ + Kernel->db->lastSlowdown = time; + Kernel->db->lastSlowdownIdle = Kernel->db->idleTime; + } + else + { + /* Compute ellapsed time in milliseconds. */ + gctUINT delta = gckOS_ProfileToMS(time - Kernel->db->lastSlowdown); + + /* Test for end of period. */ + if (delta >= gcdDYNAMIC_SPEED) + { + /* Compute number of idle milliseconds. */ + gctUINT idle = gckOS_ProfileToMS( + Kernel->db->idleTime - Kernel->db->lastSlowdownIdle); + + /* Broadcast to slow down the GPU. */ + gcmkONERROR(gckOS_BroadcastCalibrateSpeed(Kernel->os, + Kernel->hardware, + idle, + delta)); + + /* Save current time. */ + Kernel->db->lastSlowdown = time; + Kernel->db->lastSlowdownIdle = Kernel->db->idleTime; + } + } + } +#endif + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(Pointer != gcvNULL); + + /* Find the database. */ + gcmkONERROR(gckKERNEL_FindDatabase(Kernel, ProcessID, gcvFALSE, &database)); + + /* Create a new record in the database. */ + gcmkONERROR(gckKERNEL_NewRecord(Kernel, database, _GetSlot(database, Pointer), &record)); + + /* Initialize the record. */ + record->kernel = Kernel; + record->type = Type; + record->data = Pointer; + record->physical = Physical; + record->bytes = Size; + + /* Get pointer to counters. */ + switch (Type) + { + case gcvDB_VIDEO_MEMORY: + count = &database->vidMem; + break; + + case gcvDB_NON_PAGED: + count = &database->nonPaged; + break; + + case gcvDB_CONTIGUOUS: + count = &database->contiguous; + break; + + case gcvDB_MAP_MEMORY: + count = &database->mapMemory; + break; + + case gcvDB_MAP_USER_MEMORY: + count = &database->mapUserMemory; + break; + + default: + count = gcvNULL; + break; + } + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, database->counterMutex, gcvINFINITE)); + + if (count != gcvNULL) + { + /* Adjust counters. */ + count->totalBytes += Size; + count->bytes += Size; + count->allocCount++; + + if (count->bytes > count->maxBytes) + { + count->maxBytes = count->bytes; + } + } + + if (Type == gcvDB_VIDEO_MEMORY) + { + count = &database->vidMemType[vidMemType]; + + /* Adjust counters. */ + count->totalBytes += Size; + count->bytes += Size; + count->allocCount++; + + if (count->bytes > count->maxBytes) + { + count->maxBytes = count->bytes; + } + + count = &database->vidMemPool[vidMemPool]; + + /* Adjust counters. */ + count->totalBytes += Size; + count->bytes += Size; + count->allocCount++; + + if (count->bytes > count->maxBytes) + { + count->maxBytes = count->bytes; + } + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, database->counterMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_RemoveProcessDB +** +** Remove a record from a process database. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gctUINT32 ProcessID +** Process ID used to identify the database. +** +** gceDATABASE_TYPE TYPE +** Type of the record to remove. +** +** gctPOINTER Pointer +** Data of the record to remove. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_RemoveProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gceDATABASE_TYPE Type, + IN gctPOINTER Pointer + ) +{ + gceSTATUS status; + gcsDATABASE_PTR database; + gctSIZE_T bytes = 0; + gctUINT32 vidMemType; + gcePOOL vidMemPool; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d Type=%d Pointer=%p", + Kernel, ProcessID, Type, Pointer); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Pointer != gcvNULL); + + /* Decode type. */ + vidMemType = (Type & gcdDB_VIDEO_MEMORY_TYPE_MASK) >> gcdDB_VIDEO_MEMORY_TYPE_SHIFT; + vidMemPool = (Type & gcdDB_VIDEO_MEMORY_POOL_MASK) >> gcdDB_VIDEO_MEMORY_POOL_SHIFT; + + Type &= gcdDATABASE_TYPE_MASK; + + /* Find the database. */ + gcmkONERROR(gckKERNEL_FindDatabase(Kernel, ProcessID, gcvFALSE, &database)); + + /* Delete the record. */ + gcmkONERROR( + gckKERNEL_DeleteRecord(Kernel, database, Type, Pointer, &bytes)); + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, database->counterMutex, gcvINFINITE)); + + /* Update counters. */ + switch (Type) + { + case gcvDB_VIDEO_MEMORY: + database->vidMem.bytes -= bytes; + database->vidMem.freeCount++; + database->vidMemType[vidMemType].bytes -= bytes; + database->vidMemType[vidMemType].freeCount++; + database->vidMemPool[vidMemPool].bytes -= bytes; + database->vidMemPool[vidMemPool].freeCount++; + break; + + case gcvDB_NON_PAGED: + database->nonPaged.bytes -= bytes; + database->nonPaged.freeCount++; + break; + + case gcvDB_CONTIGUOUS: + database->contiguous.bytes -= bytes; + database->contiguous.freeCount++; + break; + + case gcvDB_MAP_MEMORY: + database->mapMemory.bytes -= bytes; + database->mapMemory.freeCount++; + break; + + case gcvDB_MAP_USER_MEMORY: + database->mapUserMemory.bytes -= bytes; + database->mapUserMemory.freeCount++; + break; + + default: + break; + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, database->counterMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_FindProcessDB +** +** Find a record from a process database. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gctUINT32 ProcessID +** Process ID used to identify the database. +** +** gceDATABASE_TYPE TYPE +** Type of the record to remove. +** +** gctPOINTER Pointer +** Data of the record to remove. +** +** OUTPUT: +** +** gcsDATABASE_RECORD_PTR Record +** Copy of record. +*/ +gceSTATUS +gckKERNEL_FindProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 ThreadID, + IN gceDATABASE_TYPE Type, + IN gctPOINTER Pointer, + OUT gcsDATABASE_RECORD_PTR Record + ) +{ + gceSTATUS status; + gcsDATABASE_PTR database; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d Type=%d Pointer=%p", + Kernel, ProcessID, ThreadID, Type, Pointer); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Pointer != gcvNULL); + + /* Find the database. */ + gcmkONERROR(gckKERNEL_FindDatabase(Kernel, ProcessID, gcvFALSE, &database)); + + /* Find the record. */ + gcmkONERROR( + gckKERNEL_FindRecord(Kernel, database, Type, Pointer, Record)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_DestroyProcessDB +** +** Destroy a process database. If the database contains any records, the data +** inside those records will be deleted as well. This aids in the cleanup if +** a process has died unexpectedly or has memory leaks. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gctUINT32 ProcessID +** Process ID used to identify the database. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_DestroyProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gckKERNEL kernel = Kernel; + gcsDATABASE_PTR previous = gcvNULL; + gcsDATABASE_PTR database = gcvNULL; + gcsDATABASE_PTR db = gcvNULL; + gctBOOL acquired = gcvFALSE; + gctSIZE_T slot; + gctUINT32 i; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d", Kernel, ProcessID); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + + /* Compute the hash for the database. */ + slot = ProcessID % gcmCOUNTOF(Kernel->db->db); + + /* Acquire the database mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->db->dbMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Walk the hash list. */ + for (database = Kernel->db->db[slot]; + database != gcvNULL; + database = database->next) + { + if (database->processID == ProcessID) + { + break; + } + } + + if (database) + { + gctINT32 oldVal = 0; + gcmkONERROR(gckOS_AtomDecrement(Kernel->os, database->refs, &oldVal)); + if (oldVal != 1) + { + goto OnExit; + } + + /* Mark it for delete so disallow reenter until really delete it */ + gcmkASSERT(!database->deleted); + database->deleted = gcvTRUE; + } + else + { + gcmkFATAL("%s(%d): DB destroy of Process=0x%x cannot match with creation\n", + __FUNCTION__, __LINE__, ProcessID); + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + +#if gcdCAPTURE_ONLY_MODE + gcmkPRINT("Capture only mode: The max allocation from System Pool is %llu bytes", database->vidMemPool[gcvPOOL_SYSTEM].maxBytes); +#endif + + /* Cannot remove the database from the hash list + ** since later records deinit need to access from the hash + */ + + gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + acquired = gcvFALSE; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_DATABASE, + "DB(%d): VidMem: total=%lu max=%lu", + ProcessID, database->vidMem.totalBytes, + database->vidMem.maxBytes); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_DATABASE, + "DB(%d): NonPaged: total=%lu max=%lu", + ProcessID, database->nonPaged.totalBytes, + database->nonPaged.maxBytes); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_DATABASE, + "DB(%d): Idle time=%llu", + ProcessID, Kernel->db->idleTime); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_DATABASE, + "DB(%d): Map: total=%lu max=%lu", + ProcessID, database->mapMemory.totalBytes, + database->mapMemory.maxBytes); + + { + gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_DATABASE, + "Process %d has entries in its database:", + ProcessID); + } + + for (i = 0; i < gcmCOUNTOF(database->list); i++) + { + gcsDATABASE_RECORD_PTR record, next; + + /* Walk all records. */ + for (record = database->list[i]; record != gcvNULL; record = next) + { + gctBOOL asynchronous = gcvTRUE; + gckVIDMEM_NODE nodeObject; + gctPHYS_ADDR physical; + gctUINT32 handle; + + /* Next next record. */ + next = record->next; + + /* Dispatch on record type. */ + switch (record->type) + { + case gcvDB_VIDEO_MEMORY: + gcmkERR_BREAK(gckVIDMEM_HANDLE_Lookup(record->kernel, + ProcessID, + gcmPTR2INT32(record->data), + &nodeObject)); + + /* Free the video memory. */ + gcmkVERIFY_OK(gckVIDMEM_HANDLE_Dereference(record->kernel, + ProcessID, + gcmPTR2INT32(record->data))); + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(record->kernel, + nodeObject)); + + gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_DATABASE, + "DB: VIDEO_MEMORY 0x%x (status=%d)", + record->data, status); + break; + + case gcvDB_NON_PAGED: + physical = gcmNAME_TO_PTR(record->physical); + + /* Free the non paged memory. */ + status = gckOS_FreeNonPagedMemory(Kernel->os, + physical, + record->data, + record->bytes); + + gcmRELEASE_NAME(record->physical); + + gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_DATABASE, + "DB: NON_PAGED 0x%x, bytes=%lu (status=%d)", + record->data, record->bytes, status); + break; + + case gcvDB_SIGNAL: +#if USE_NEW_LINUX_SIGNAL + status = gcvSTATUS_NOT_SUPPORTED; +#else + /* Free the user signal. */ + status = gckOS_DestroyUserSignal(Kernel->os, + gcmPTR2INT32(record->data)); +#endif /* USE_NEW_LINUX_SIGNAL */ + + gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_DATABASE, + "DB: SIGNAL %d (status=%d)", + (gctINT)(gctUINTPTR_T)record->data, status); + break; + + case gcvDB_VIDEO_MEMORY_LOCKED: + handle = gcmPTR2INT32(record->data); + + gcmkERR_BREAK(gckVIDMEM_HANDLE_Lookup(record->kernel, + ProcessID, + handle, + &nodeObject)); + + /* Unlock CPU. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + record->kernel, nodeObject, ProcessID, gcvTRUE, gcvFALSE)); + + /* Unlock what we still locked */ + status = gckVIDMEM_NODE_Unlock(record->kernel, + nodeObject, + ProcessID, + &asynchronous); + + { + /* Deref handle. */ + gcmkVERIFY_OK(gckVIDMEM_HANDLE_Dereference(record->kernel, + ProcessID, + handle)); + + if (gcmIS_SUCCESS(status) && (gcvTRUE == asynchronous)) + { + /* Schedule unlock: will unlock and deref node later. */ + status = gckEVENT_Unlock(record->kernel->eventObj, + gcvKERNEL_PIXEL, + nodeObject); + } + else + { + /* Deref node */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(record->kernel, + nodeObject)); + } + } + + gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_DATABASE, + "DB: VIDEO_MEMORY_LOCKED 0x%x (status=%d)", + record->data, status); + break; + + case gcvDB_CONTEXT: + status = gckCOMMAND_Detach(record->kernel->command, gcmNAME_TO_PTR(record->data)); + gcmRELEASE_NAME(record->data); + + gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_DATABASE, + "DB: CONTEXT 0x%x (status=%d)", + record->data, status); + break; + + case gcvDB_MAP_MEMORY: + /* Unmap memory. */ + status = gckKERNEL_UnmapMemory(record->kernel, + record->physical, + record->bytes, + record->data, + ProcessID); + + gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_DATABASE, + "DB: MAP MEMORY %d (status=%d)", + gcmPTR2INT32(record->data), status); + break; + + case gcvDB_SHBUF: + /* Free shared buffer. */ + status = gckKERNEL_DestroyShBuffer(record->kernel, + (gctSHBUF) record->data); + + gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_DATABASE, + "DB: SHBUF %u (status=%d)", + (gctUINT32)(gctUINTPTR_T) record->data, status); + break; + + default: + gcmkTRACE_ZONE(gcvLEVEL_ERROR, gcvZONE_DATABASE, + "DB: Correcupted record=0x%08x type=%d", + record, record->type); + break; + } + + /* Delete the record. */ + gcmkONERROR(gckKERNEL_DeleteRecord(Kernel, + database, + record->type, + record->data, + gcvNULL)); + } + } + + gcmkONERROR(gckKERNEL_DestroyProcessReservedUserMap(Kernel, ProcessID)); + + /* Acquire the database mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->db->dbMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Walk the hash list. */ + for (db = Kernel->db->db[slot]; + db != gcvNULL; + db = db->next) + { + if (db->processID == ProcessID) + { + break; + } + previous = db; + } + + if (db != database || !db->deleted) + { + gcmkFATAL("%s(%d): DB of Process=0x%x corrupted after found in deletion\n", + __FUNCTION__, __LINE__, ProcessID); + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + + /* Remove the database from the hash list. */ + if (previous) + { + previous->next = database->next; + } + else + { + Kernel->db->db[slot] = database->next; + } + + /* Deinit current database. */ + gcmkVERIFY_OK(gckKERNEL_DeinitDatabase(Kernel, database)); + + if (Kernel->db->lastDatabase) + { + /* Insert last database to the free list. */ + Kernel->db->lastDatabase->next = Kernel->db->freeDatabase; + Kernel->db->freeDatabase = Kernel->db->lastDatabase; + } + + /* Update last database to current one. */ + Kernel->db->lastDatabase = database; + +OnError: +OnExit: + if (acquired) + { + /* Release the database mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + } + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckKERNEL_QueryProcessDB +** +** Query a process database for the current usage of a particular record type. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to a gckKERNEL object. +** +** gctUINT32 ProcessID +** Process ID used to identify the database. +** +** gctBOOL LastProcessID +** gcvTRUE if searching for the last known process ID. gcvFALSE if +** we need to search for the process ID specified by the ProcessID +** argument. +** +** gceDATABASE_TYPE Type +** Type of the record to query. +** +** OUTPUT: +** +** gcuDATABASE_INFO * Info +** Pointer to a variable that receives the requested information. +*/ +gceSTATUS +gckKERNEL_QueryProcessDB( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctBOOL LastProcessID, + IN gceDATABASE_TYPE Type, + OUT gcuDATABASE_INFO * Info + ) +{ + gceSTATUS status; + gcsDATABASE_PTR database; + gcePOOL vidMemPool; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d Type=%d Info=%p", + Kernel, ProcessID, Type, Info); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Info != gcvNULL); + + /* Deocde pool. */ + vidMemPool = (Type & gcdDB_VIDEO_MEMORY_POOL_MASK) >> gcdDB_VIDEO_MEMORY_POOL_SHIFT; + + Type &= gcdDATABASE_TYPE_MASK; + + /* Find the database. */ + gcmkONERROR(gckKERNEL_FindDatabase(Kernel, ProcessID, LastProcessID, &database)); + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, database->counterMutex, gcvINFINITE)); + + /* Get pointer to counters. */ + switch (Type) + { + case gcvDB_VIDEO_MEMORY: + if (vidMemPool != gcvPOOL_UNKNOWN) + { + gckOS_MemCopy(&Info->counters, + &database->vidMemPool[vidMemPool], + gcmSIZEOF(database->vidMemPool[vidMemPool])); + } + else + { + gckOS_MemCopy(&Info->counters, + &database->vidMem, + gcmSIZEOF(database->vidMem)); + } + break; + + case gcvDB_NON_PAGED: + gckOS_MemCopy(&Info->counters, + &database->nonPaged, + gcmSIZEOF(database->vidMem)); + break; + + case gcvDB_CONTIGUOUS: + gckOS_MemCopy(&Info->counters, + &database->contiguous, + gcmSIZEOF(database->vidMem)); + break; + + case gcvDB_IDLE: + Info->time = Kernel->db->idleTime; + Kernel->db->idleTime = 0; + break; + + case gcvDB_MAP_MEMORY: + gckOS_MemCopy(&Info->counters, + &database->mapMemory, + gcmSIZEOF(database->mapMemory)); + break; + + default: + break; + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, database->counterMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_FindHandleDatbase( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + OUT gctPOINTER * HandleDatabase, + OUT gctPOINTER * HandleDatabaseMutex + ) +{ + gceSTATUS status; + gcsDATABASE_PTR database; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d", + Kernel, ProcessID); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + + /* Find the database. */ + gcmkONERROR(gckKERNEL_FindDatabase(Kernel, ProcessID, gcvFALSE, &database)); + + *HandleDatabase = database->handleDatabase; + *HandleDatabaseMutex = database->handleDatabaseMutex; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_DumpProcessDB( + IN gckKERNEL Kernel + ) +{ + gcsDATABASE_PTR database; + gctINT i, pid; + gctUINT8 name[24]; + + gcmkHEADER_ARG("Kernel=%p", Kernel); + + /* Acquire the database mutex. */ + gcmkVERIFY_OK( + gckOS_AcquireMutex(Kernel->os, Kernel->db->dbMutex, gcvINFINITE)); + + gcmkPRINT("**************************\n"); + gcmkPRINT("*** PROCESS DB DUMP ***\n"); + gcmkPRINT("**************************\n"); + + gcmkPRINT_N(8, "%-8s%s\n", "PID", "NAME"); + /* Walk the databases. */ + for (i = 0; i < gcmCOUNTOF(Kernel->db->db); ++i) + { + for (database = Kernel->db->db[i]; + database != gcvNULL; + database = database->next) + { + pid = database->processID; + + gcmkVERIFY_OK(gckOS_ZeroMemory(name, gcmSIZEOF(name))); + + gcmkVERIFY_OK(gckOS_GetProcessNameByPid(pid, gcmSIZEOF(name), name)); + + gcmkPRINT_N(8, "%-8d%s\n", pid, name); + } + } + + /* Release the database mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->db->dbMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +void +_DumpCounter( + IN gcsDATABASE_COUNTERS * Counter, + IN gctCONST_STRING Name + ) +{ + gcmkPRINT("%s:", Name); + gcmkPRINT(" Currently allocated : %10lld", Counter->bytes); + gcmkPRINT(" Maximum allocated : %10lld", Counter->maxBytes); + gcmkPRINT(" Total allocated : %10lld", Counter->totalBytes); +} + +gceSTATUS +gckKERNEL_DumpVidMemUsage( + IN gckKERNEL Kernel, + IN gctINT32 ProcessID + ) +{ + gceSTATUS status; + gcsDATABASE_PTR database; + gcsDATABASE_COUNTERS * counter; + gctUINT32 i = 0; + + static gctCONST_STRING vidmemTypes[] = { + "GENERIC", + "INDEX", + "VERTEX", + "TEXTURE", + "RENDER_TARGET", + "DEPTH", + "BITMAP", + "TILE_STATUS", + "IMAGE", + "MASK", + "SCISSOR", + "HIERARCHICAL_DEPTH", + "ICACHE", + "TXDESC", + "FENCE", + "TFBHEADER", + "COMMAND", + }; + + gcmkHEADER_ARG("Kernel=%p ProcessID=%d", + Kernel, ProcessID); + + gcmSTATIC_ASSERT(gcmCOUNTOF(vidmemTypes) == gcvVIDMEM_TYPE_COUNT, + "Video memory type mismatch"); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + + /* Find the database. */ + gcmkONERROR( + gckKERNEL_FindDatabase(Kernel, ProcessID, gcvFALSE, &database)); + + gcmkPRINT("VidMem Usage (Process %d):", ProcessID); + + /* Get pointer to counters. */ + counter = &database->vidMem; + + _DumpCounter(counter, "Total Video Memory"); + + for (i = 0; i < gcvVIDMEM_TYPE_COUNT; i++) + { + counter = &database->vidMemType[i]; + + _DumpCounter(counter, vidmemTypes[i]); + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +}
diff --git a/hal/kernel/gc_hal_kernel_debug.c b/hal/kernel/gc_hal_kernel_debug.c new file mode 100644 index 0000000..290f608 --- /dev/null +++ b/hal/kernel/gc_hal_kernel_debug.c
@@ -0,0 +1,1312 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" +#include <gc_hal_kernel_debug.h> + +/******************************************************************************\ +******************************** Debug Variables ******************************* +\******************************************************************************/ + +static gceSTATUS _lastError = gcvSTATUS_OK; +static gctUINT32 _debugLevel = gcvLEVEL_ERROR; +/* +_debugZones config value +Please Reference define in gc_hal_base.h +*/ +static gctUINT32 _debugZones = gcdZONE_NONE; + +/******************************************************************************\ +********************************* Debug Switches ******************************* +\******************************************************************************/ + +/* + gcdTHREAD_BUFFERS + + When greater then one, will accumulate messages from the specified number + of threads in separate output buffers. +*/ +#define gcdTHREAD_BUFFERS 1 + +/* + gcdSHOW_LINE_NUMBER + + When enabledm each print statement will be preceeded with the current + line number. +*/ +#define gcdSHOW_LINE_NUMBER 0 + +/* + gcdSHOW_PROCESS_ID + + When enabledm each print statement will be preceeded with the current + process ID. +*/ +#define gcdSHOW_PROCESS_ID 0 + +/* + gcdSHOW_THREAD_ID + + When enabledm each print statement will be preceeded with the current + thread ID. +*/ +#define gcdSHOW_THREAD_ID 0 + +/* + gcdSHOW_TIME + + When enabled each print statement will be preceeded with the current + high-resolution time. +*/ +#define gcdSHOW_TIME 0 + + +/******************************************************************************\ +****************************** Miscellaneous Macros **************************** +\******************************************************************************/ + +#if gcdSHOW_TIME || gcdSHOW_LINE_NUMBER || gcdSHOW_PROCESS_ID || gcdSHOW_THREAD_ID +# define gcdHAVEPREFIX 1 +#else +# define gcdHAVEPREFIX 0 +#endif + +/******************************************************************************\ +****************************** Private Structures ****************************** +\******************************************************************************/ + +typedef struct _gcsBUFFERED_OUTPUT * gcsBUFFERED_OUTPUT_PTR; +typedef struct _gcsBUFFERED_OUTPUT +{ +#if gcdTHREAD_BUFFERS > 1 + gctUINT32 threadID; +#endif + +#if gcdSHOW_LINE_NUMBER + gctUINT lineNumber; +#endif + + gctINT indent; + + gcsBUFFERED_OUTPUT_PTR prev; + gcsBUFFERED_OUTPUT_PTR next; +} +gcsBUFFERED_OUTPUT; + +static gcsBUFFERED_OUTPUT _outputBuffer[gcdTHREAD_BUFFERS]; +static gcsBUFFERED_OUTPUT_PTR _outputBufferHead = gcvNULL; +static gcsBUFFERED_OUTPUT_PTR _outputBufferTail = gcvNULL; + +/******************************************************************************\ +******************************* Printing Functions ***************************** +\******************************************************************************/ + +#if gcdHAVEPREFIX + +#if gcdSHOW_TIME +static gcmINLINE gctUINT64 +_GetTime( + void + ) +{ + gctUINT64 time; + gckOS_GetProfileTick(&time); + return time; +} +# define gcdPREFIX_LEADER 1 +# define gcdTIMEFORMAT "%18lld" +# define gcdTIMEVALUE ,_GetTime() +# else +# define gcdTIMEFORMAT +# define gcdTIMEVALUE +# endif + +#if gcdSHOW_LINE_NUMBER +#ifndef gcdPREFIX_LEADER +# define gcdPREFIX_LEADER 1 +# define gcdNUMFORMAT "%8u" +# else +# define gcdNUMFORMAT ", %8u" +# endif +# define gcdNUMVALUE ,OutputBuffer->lineNumber +# else +# define gcdNUMFORMAT +# define gcdNUMVALUE +# endif + +#if gcdSHOW_PROCESS_ID +#ifndef gcdPREFIX_LEADER +# define gcdPREFIX_LEADER 1 +# define gcdPIDFORMAT "pid=%5u" +# else +# define gcdPIDFORMAT ", pid=%5u" +# endif +# define gcdPIDVALUE ,gcmkGETPROCESSID() +# else +# define gcdPIDFORMAT +# define gcdPIDVALUE +# endif + +#if gcdSHOW_THREAD_ID +#ifndef gcdPREFIX_LEADER +# define gcdPREFIX_LEADER 1 +# define gcdTIDFORMAT "tid=%5u" +# else +# define gcdTIDFORMAT ", tid=%5u" +# endif +# define gcdTIDVALUE ,gcmkGETTHREADID() +# else +# define gcdTIDFORMAT +# define gcdTIDVALUE +# endif + +static gctUINT +_PrintPrefix( + IN gcsBUFFERED_OUTPUT_PTR OutputBuffer, + IN char Buffer[], + IN gctUINT Size + ) +{ + gctINT len; + + /* Format the string. */ + len = gcmkSPRINTF(Buffer, + Size, + "[" gcdTIMEFORMAT gcdNUMFORMAT gcdPIDFORMAT gcdTIDFORMAT "] " + gcdTIMEVALUE gcdNUMVALUE gcdPIDVALUE gcdTIDVALUE); + + if (len > 0) + { + Buffer[len] = '\0'; + return (gctUINT)len; + } + + return 0; +} +#endif + +static int +_AppendIndent( + IN gctINT Indent, + IN char * Buffer, + IN int BufferSize + ) +{ + gctINT i; + + gctINT len = 0; + gctINT indent = Indent % 40; + + for (i = 0; i < indent; i += 1) + { + Buffer[len++] = ' '; + } + + if (indent != Indent) + { + len += gcmkSPRINTF( + Buffer + len, BufferSize - len, " <%d> ", Indent + ); + + Buffer[len] = '\0'; + } + + return len; +} + +static gctUINT +_PrintString( + IN gcsBUFFERED_OUTPUT_PTR OutputBuffer, + IN gctINT Indent, + IN gctCONST_STRING Message, + IN gctPOINTER Data, + IN char Buffer[], + IN gctUINT Size + ) +{ + gctINT len; + + /* Append the indent string. */ + len = _AppendIndent(Indent, Buffer, Size); + + /* Format the string. */ + len += gcmkVSPRINTF(Buffer + len, Size - len, Message, Data); + Buffer[len] = '\0'; + + /* Add end-of-line if missing. */ + if (Buffer[len - 1] != '\n') + { + Buffer[len++] = '\n'; + Buffer[len] = '\0'; + } + + return (gctUINT)len; +} + +/******************************************************************************\ +******************************* Private Functions ****************************** +\******************************************************************************/ + +static gcmINLINE void +_InitBuffers( + void + ) +{ + int i; + + if (_outputBufferHead == gcvNULL) + { + for (i = 0; i < gcdTHREAD_BUFFERS; i += 1) + { + if (_outputBufferTail == gcvNULL) + { + _outputBufferHead = &_outputBuffer[i]; + } + else + { + _outputBufferTail->next = &_outputBuffer[i]; + } + +#if gcdTHREAD_BUFFERS > 1 + _outputBuffer[i].threadID = ~0U; +#endif + + _outputBuffer[i].prev = _outputBufferTail; + _outputBuffer[i].next = gcvNULL; + + _outputBufferTail = &_outputBuffer[i]; + } + } +} + +static gcmINLINE gcsBUFFERED_OUTPUT_PTR +_GetOutputBuffer( + void + ) +{ + gcsBUFFERED_OUTPUT_PTR outputBuffer; + +#if gcdTHREAD_BUFFERS > 1 + /* Get the current thread ID. */ + gctUINT32 ThreadID = gcmkGETTHREADID(); + + /* Locate the output buffer for the thread. */ + outputBuffer = _outputBufferHead; + + while (outputBuffer != gcvNULL) + { + if (outputBuffer->threadID == ThreadID) + { + break; + } + + outputBuffer = outputBuffer->next; + } + + /* No matching buffer found? */ + if (outputBuffer == gcvNULL) + { + /* Get the tail for the buffer. */ + outputBuffer = _outputBufferTail; + + /* Move it to the head. */ + _outputBufferTail = _outputBufferTail->prev; + _outputBufferTail->next = gcvNULL; + + outputBuffer->prev = gcvNULL; + outputBuffer->next = _outputBufferHead; + + _outputBufferHead->prev = outputBuffer; + _outputBufferHead = outputBuffer; + + /* Reset the buffer. */ + outputBuffer->threadID = ThreadID; +#if gcdSHOW_LINE_NUMBER + outputBuffer->lineNumber = 0; +# endif + } +#else + outputBuffer = _outputBufferHead; +#endif + + return outputBuffer; +} + +static void +_Print( + IN gctCONST_STRING Message, + IN gctARGUMENTS * Arguments + ) +{ + gcsBUFFERED_OUTPUT_PTR outputBuffer; + char buffer[256]; + char *ptr = buffer; + gctINT len = 0; + static gcmkDECLARE_MUTEX(printMutex); + + gcmkMUTEX_LOCK(printMutex); + + /* Initialize output buffer list. */ + _InitBuffers(); + + /* Locate the proper output buffer. */ + outputBuffer = _GetOutputBuffer(); + + /* Print prefix. */ +#if gcdHAVEPREFIX +#if gcdSHOW_LINE_NUMBER + /* Update the line number. */ + outputBuffer->lineNumber += 1; +# endif + + /* Print the prefix. */ + len = _PrintPrefix(outputBuffer, buffer, gcmSIZEOF(buffer)); + ptr += len; +#endif + + /* Form the indent string. */ + if (Message[0] == '-' && Message[1] == '-') + { + outputBuffer->indent -= 2; + } + + /* Print the message. */ + len += _PrintString( + outputBuffer, outputBuffer->indent, + Message, ((gctPOINTER) Arguments), + ptr, gcmSIZEOF(buffer) - outputBuffer->indent - len + ); + + gcmkOUTPUT_STRING(buffer); + + /* Check increasing indent. */ + if (Message[0] == '+' && Message[1] == '+') + { + outputBuffer->indent += 2; + } + + gcmkMUTEX_UNLOCK(printMutex); +} + + +/******************************************************************************\ +********************************* Debug Macros ********************************* +\******************************************************************************/ + +#ifdef __QNXNTO__ + +extern volatile unsigned g_nQnxInIsrs; + +#define gcmDEBUGPRINT(Message) \ +{ \ + if (atomic_add_value(&g_nQnxInIsrs, 1) == 0) \ + { \ + gctARGUMENTS __arguments__; \ + gcmkARGUMENTS_START(__arguments__, Message); \ + _Print(Message, &__arguments__); \ + gcmkARGUMENTS_END(__arguments__); \ + } \ + atomic_sub(&g_nQnxInIsrs, 1); \ +} + +#elif defined(__VXWORKS__) +#define gcmDEBUGPRINT(Message) \ +{ \ + printf(Message); \ +} + +#else + +#define gcmDEBUGPRINT(Message) \ +{ \ + gctARGUMENTS __arguments__; \ + gcmkARGUMENTS_START(__arguments__, Message); \ + _Print(Message, &__arguments__); \ + gcmkARGUMENTS_END(__arguments__); \ +} + +#endif + +/******************************************************************************\ +********************************** Debug Code ********************************** +\******************************************************************************/ + +/******************************************************************************* +** +** gckOS_Print +** +** Send a message to the debugger. +** +** INPUT: +** +** gctCONST_STRING Message +** Pointer to message. +** +** ... +** Optional arguments. +** +** OUTPUT: +** +** Nothing. +*/ + +void +gckOS_Print( + IN gctCONST_STRING Message, + ... + ) +{ + gcmDEBUGPRINT(Message); +} + +/******************************************************************************* +** +** gckOS_DebugTrace +** +** Send a leveled message to the debugger. +** +** INPUT: +** +** gctUINT32 Level +** Debug level of message. +** +** gctCONST_STRING Message +** Pointer to message. +** +** ... +** Optional arguments. +** +** OUTPUT: +** +** Nothing. +*/ + +void +gckOS_DebugTrace( + IN gctUINT32 Level, + IN gctCONST_STRING Message, + ... + ) +{ + if (Level > _debugLevel) + { + return; + } + + gcmDEBUGPRINT(Message); +} + +/******************************************************************************* +** +** gckOS_DebugTraceZone +** +** Send a leveled and zoned message to the debugger. +** +** INPUT: +** +** gctUINT32 Level +** Debug level for message. +** +** gctUINT32 Zone +** Debug zone for message. +** +** gctCONST_STRING Message +** Pointer to message. +** +** ... +** Optional arguments. +** +** OUTPUT: +** +** Nothing. +*/ + +void +gckOS_DebugTraceZone( + IN gctUINT32 Level, + IN gctUINT32 Zone, + IN gctCONST_STRING Message, + ... + ) +{ + if ((Level > _debugLevel) || !(Zone & _debugZones)) + { + return; + } + + gcmDEBUGPRINT(Message); +} + +/******************************************************************************* +** +** gckOS_DebugBreak +** +** Break into the debugger. +** +** INPUT: +** +** Nothing. +** +** OUTPUT: +** +** Nothing. +*/ +void +gckOS_DebugBreak( + void + ) +{ + gckOS_DebugTrace(gcvLEVEL_ERROR, "%s(%d)", __FUNCTION__, __LINE__); +} + +/******************************************************************************* +** +** gckOS_DebugFatal +** +** Send a message to the debugger and break into the debugger. +** +** INPUT: +** +** gctCONST_STRING Message +** Pointer to message. +** +** ... +** Optional arguments. +** +** OUTPUT: +** +** Nothing. +*/ +void +gckOS_DebugFatal( + IN gctCONST_STRING Message, + ... + ) +{ + gcmkPRINT_VERSION(); + gcmDEBUGPRINT(Message); + + /* Break into the debugger. */ + gckOS_DebugBreak(); +} + +/******************************************************************************* +** +** gckOS_SetDebugLevel +** +** Set the debug level. +** +** INPUT: +** +** gctUINT32 Level +** New debug level. +** +** OUTPUT: +** +** Nothing. +*/ + +void +gckOS_SetDebugLevel( + IN gctUINT32 Level + ) +{ + _debugLevel = Level; +} + +/******************************************************************************* +** +** gckOS_SetDebugZone +** +** Set the debug zone. +** +** INPUT: +** +** gctUINT32 Zone +** New debug zone. +** +** OUTPUT: +** +** Nothing. +*/ +void +gckOS_SetDebugZone( + IN gctUINT32 Zone + ) +{ + _debugZones = Zone; +} + +/******************************************************************************* +** +** gckOS_SetDebugLevelZone +** +** Set the debug level and zone. +** +** INPUT: +** +** gctUINT32 Level +** New debug level. +** +** gctUINT32 Zone +** New debug zone. +** +** OUTPUT: +** +** Nothing. +*/ + +void +gckOS_SetDebugLevelZone( + IN gctUINT32 Level, + IN gctUINT32 Zone + ) +{ + _debugLevel = Level; + _debugZones = Zone; +} + +/******************************************************************************* +** +** gckOS_SetDebugZones +** +** Enable or disable debug zones. +** +** INPUT: +** +** gctUINT32 Zones +** Debug zones to enable or disable. +** +** gctBOOL Enable +** Set to gcvTRUE to enable the zones (or the Zones with the current +** zones) or gcvFALSE to disable the specified Zones. +** +** OUTPUT: +** +** Nothing. +*/ + +void +gckOS_SetDebugZones( + IN gctUINT32 Zones, + IN gctBOOL Enable + ) +{ + if (Enable) + { + /* Enable the zones. */ + _debugZones |= Zones; + } + else + { + /* Disable the zones. */ + _debugZones &= ~Zones; + } +} + +/******************************************************************************* +** +** gckOS_Verify +** +** Called to verify the result of a function call. +** +** INPUT: +** +** gceSTATUS Status +** Function call result. +** +** OUTPUT: +** +** Nothing. +*/ + +void +gckOS_Verify( + IN gceSTATUS status + ) +{ + _lastError = status; +} + +gctCONST_STRING +gckOS_DebugStatus2Name( + gceSTATUS status + ) +{ + switch (status) + { + case gcvSTATUS_OK: + return "gcvSTATUS_OK"; + case gcvSTATUS_TRUE: + return "gcvSTATUS_TRUE"; + case gcvSTATUS_NO_MORE_DATA: + return "gcvSTATUS_NO_MORE_DATA"; + case gcvSTATUS_CACHED: + return "gcvSTATUS_CACHED"; + case gcvSTATUS_MIPMAP_TOO_LARGE: + return "gcvSTATUS_MIPMAP_TOO_LARGE"; + case gcvSTATUS_NAME_NOT_FOUND: + return "gcvSTATUS_NAME_NOT_FOUND"; + case gcvSTATUS_NOT_OUR_INTERRUPT: + return "gcvSTATUS_NOT_OUR_INTERRUPT"; + case gcvSTATUS_MISMATCH: + return "gcvSTATUS_MISMATCH"; + case gcvSTATUS_MIPMAP_TOO_SMALL: + return "gcvSTATUS_MIPMAP_TOO_SMALL"; + case gcvSTATUS_LARGER: + return "gcvSTATUS_LARGER"; + case gcvSTATUS_SMALLER: + return "gcvSTATUS_SMALLER"; + case gcvSTATUS_CHIP_NOT_READY: + return "gcvSTATUS_CHIP_NOT_READY"; + case gcvSTATUS_NEED_CONVERSION: + return "gcvSTATUS_NEED_CONVERSION"; + case gcvSTATUS_SKIP: + return "gcvSTATUS_SKIP"; + case gcvSTATUS_DATA_TOO_LARGE: + return "gcvSTATUS_DATA_TOO_LARGE"; + case gcvSTATUS_INVALID_CONFIG: + return "gcvSTATUS_INVALID_CONFIG"; + case gcvSTATUS_CHANGED: + return "gcvSTATUS_CHANGED"; + case gcvSTATUS_NOT_SUPPORT_DITHER: + return "gcvSTATUS_NOT_SUPPORT_DITHER"; + + case gcvSTATUS_INVALID_ARGUMENT: + return "gcvSTATUS_INVALID_ARGUMENT"; + case gcvSTATUS_INVALID_OBJECT: + return "gcvSTATUS_INVALID_OBJECT"; + case gcvSTATUS_OUT_OF_MEMORY: + return "gcvSTATUS_OUT_OF_MEMORY"; + case gcvSTATUS_MEMORY_LOCKED: + return "gcvSTATUS_MEMORY_LOCKED"; + case gcvSTATUS_MEMORY_UNLOCKED: + return "gcvSTATUS_MEMORY_UNLOCKED"; + case gcvSTATUS_HEAP_CORRUPTED: + return "gcvSTATUS_HEAP_CORRUPTED"; + case gcvSTATUS_GENERIC_IO: + return "gcvSTATUS_GENERIC_IO"; + case gcvSTATUS_INVALID_ADDRESS: + return "gcvSTATUS_INVALID_ADDRESS"; + case gcvSTATUS_CONTEXT_LOSSED: + return "gcvSTATUS_CONTEXT_LOSSED"; + case gcvSTATUS_TOO_COMPLEX: + return "gcvSTATUS_TOO_COMPLEX"; + case gcvSTATUS_BUFFER_TOO_SMALL: + return "gcvSTATUS_BUFFER_TOO_SMALL"; + case gcvSTATUS_INTERFACE_ERROR: + return "gcvSTATUS_INTERFACE_ERROR"; + case gcvSTATUS_NOT_SUPPORTED: + return "gcvSTATUS_NOT_SUPPORTED"; + case gcvSTATUS_MORE_DATA: + return "gcvSTATUS_MORE_DATA"; + case gcvSTATUS_TIMEOUT: + return "gcvSTATUS_TIMEOUT"; + case gcvSTATUS_OUT_OF_RESOURCES: + return "gcvSTATUS_OUT_OF_RESOURCES"; + case gcvSTATUS_INVALID_DATA: + return "gcvSTATUS_INVALID_DATA"; + case gcvSTATUS_INVALID_MIPMAP: + return "gcvSTATUS_INVALID_MIPMAP"; + case gcvSTATUS_NOT_FOUND: + return "gcvSTATUS_NOT_FOUND"; + case gcvSTATUS_NOT_ALIGNED: + return "gcvSTATUS_NOT_ALIGNED"; + case gcvSTATUS_INVALID_REQUEST: + return "gcvSTATUS_INVALID_REQUEST"; + case gcvSTATUS_GPU_NOT_RESPONDING: + return "gcvSTATUS_GPU_NOT_RESPONDING"; + case gcvSTATUS_TIMER_OVERFLOW: + return "gcvSTATUS_TIMER_OVERFLOW"; + case gcvSTATUS_VERSION_MISMATCH: + return "gcvSTATUS_VERSION_MISMATCH"; + case gcvSTATUS_LOCKED: + return "gcvSTATUS_LOCKED"; + case gcvSTATUS_INTERRUPTED: + return "gcvSTATUS_INTERRUPTED"; + case gcvSTATUS_DEVICE: + return "gcvSTATUS_DEVICE"; + case gcvSTATUS_NOT_MULTI_PIPE_ALIGNED: + return "gcvSTATUS_NOT_MULTI_PIPE_ALIGNED"; + + /* Linker errors. */ + case gcvSTATUS_GLOBAL_TYPE_MISMATCH: + return "gcvSTATUS_GLOBAL_TYPE_MISMATCH"; + case gcvSTATUS_TOO_MANY_ATTRIBUTES: + return "gcvSTATUS_TOO_MANY_ATTRIBUTES"; + case gcvSTATUS_TOO_MANY_UNIFORMS: + return "gcvSTATUS_TOO_MANY_UNIFORMS"; + case gcvSTATUS_TOO_MANY_VARYINGS: + return "gcvSTATUS_TOO_MANY_VARYINGS"; + case gcvSTATUS_UNDECLARED_VARYING: + return "gcvSTATUS_UNDECLARED_VARYING"; + case gcvSTATUS_VARYING_TYPE_MISMATCH: + return "gcvSTATUS_VARYING_TYPE_MISMATCH"; + case gcvSTATUS_MISSING_MAIN: + return "gcvSTATUS_MISSING_MAIN"; + case gcvSTATUS_NAME_MISMATCH: + return "gcvSTATUS_NAME_MISMATCH"; + case gcvSTATUS_INVALID_INDEX: + return "gcvSTATUS_INVALID_INDEX"; + case gcvSTATUS_UNIFORM_MISMATCH: + return "gcvSTATUS_UNIFORM_MISMATCH"; + case gcvSTATUS_UNSAT_LIB_SYMBOL: + return "gcvSTATUS_UNSAT_LIB_SYMBOL"; + case gcvSTATUS_TOO_MANY_SHADERS: + return "gcvSTATUS_TOO_MANY_SHADERS"; + case gcvSTATUS_LINK_INVALID_SHADERS: + return "gcvSTATUS_LINK_INVALID_SHADERS"; + case gcvSTATUS_CS_NO_WORKGROUP_SIZE: + return "gcvSTATUS_CS_NO_WORKGROUP_SIZE"; + case gcvSTATUS_LINK_LIB_ERROR: + return "gcvSTATUS_LINK_LIB_ERROR"; + case gcvSTATUS_SHADER_VERSION_MISMATCH: + return "gcvSTATUS_SHADER_VERSION_MISMATCH"; + case gcvSTATUS_TOO_MANY_INSTRUCTION: + return "gcvSTATUS_TOO_MANY_INSTRUCTION"; + case gcvSTATUS_SSBO_MISMATCH: + return "gcvSTATUS_SSBO_MISMATCH"; + case gcvSTATUS_TOO_MANY_OUTPUT: + return "gcvSTATUS_TOO_MANY_OUTPUT"; + case gcvSTATUS_TOO_MANY_INPUT: + return "gcvSTATUS_TOO_MANY_INPUT"; + case gcvSTATUS_NOT_SUPPORT_CL: + return "gcvSTATUS_NOT_SUPPORT_CL"; + case gcvSTATUS_NOT_SUPPORT_INTEGER: + return "gcvSTATUS_NOT_SUPPORT_INTEGER"; + case gcvSTATUS_UNIFORM_TYPE_MISMATCH: + return "gcvSTATUS_UNIFORM_TYPE_MISMATCH"; + case gcvSTATUS_MISSING_PRIMITIVE_TYPE: + return "gcvSTATUS_MISSING_PRIMITIVE_TYPE"; + case gcvSTATUS_MISSING_OUTPUT_VERTEX_COUNT: + return "gcvSTATUS_MISSING_OUTPUT_VERTEX_COUNT"; + case gcvSTATUS_NON_INVOCATION_ID_AS_INDEX: + return "gcvSTATUS_NON_INVOCATION_ID_AS_INDEX"; + case gcvSTATUS_INPUT_ARRAY_SIZE_MISMATCH: + return "gcvSTATUS_INPUT_ARRAY_SIZE_MISMATCH"; + case gcvSTATUS_OUTPUT_ARRAY_SIZE_MISMATCH: + return "gcvSTATUS_OUTPUT_ARRAY_SIZE_MISMATCH"; + + /* Compiler errors. */ + case gcvSTATUS_COMPILER_FE_PREPROCESSOR_ERROR: + return "gcvSTATUS_COMPILER_FE_PREPROCESSOR_ERROR"; + case gcvSTATUS_COMPILER_FE_PARSER_ERROR: + return "gcvSTATUS_COMPILER_FE_PARSER_ERROR"; + default: + return "nil"; + } +} + +/******************************************************************************* +***** Kernel Dump ************************************************************** +*******************************************************************************/ + +/* + * TODO: Dump to file is only valid in linux currently. + */ +#ifndef gcmkDUMP_STRING +# define gcmkDUMP_STRING(os, s) gcmkOUTPUT_STRING((s)) +#endif + +static gcmkDECLARE_MUTEX(_dumpMutex); +static gctCHAR _dumpStorage[512]; + +/******************************************************************************* +** +** gckOS_Dump +** +** Formated print string to dump pool. +** +** INPUT: +** +** gctCONST_STRING Format +** String format. +** +** OUTPUT: +** +** Nothing. +*/ +void +gckOS_Dump( + IN gckOS Os, + IN gctCONST_STRING Format, + ... + ) +{ + char buffer[256]; + gctINT len; + gctARGUMENTS args; + + gcmkARGUMENTS_START(args, Format); + len = gcmkVSPRINTF(buffer, gcmSIZEOF(buffer) - 2, Format, &args); + gcmkARGUMENTS_END(args); + + if (len > 0) + { + if (buffer[len - 1] != '\n') + { + buffer[len] = '\n'; + buffer[len + 1] = '\0'; + } + + gcmkMUTEX_LOCK(_dumpMutex); + gcmkDUMP_STRING(Os, buffer); + gcmkMUTEX_UNLOCK(_dumpMutex); + } +} + +static void +_DumpUserString( + IN gckOS Os, + IN gctPOINTER UserStr, + IN gctSIZE_T Size + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctSIZE_T offset = 0; + gctSIZE_T length = 0; + gctBOOL needCopy = gcvTRUE; + const gctSIZE_T maxLength = gcmSIZEOF(_dumpStorage) - 1; + + gcmkVERIFY_OK(gckOS_QueryNeedCopy(Os, 0, &needCopy)); + + gcmkMUTEX_LOCK(_dumpMutex); + + while (offset < Size) + { + length = maxLength < (Size - offset) ? maxLength : (Size - offset); + + /* Copy or map from user. */ + if (needCopy) + { + gcmkONERROR(gckOS_CopyFromUserData( + Os, + _dumpStorage, + UserStr, + length + )); + } + else + { + gctPOINTER ptr = gcvNULL; + + gcmkONERROR(gckOS_MapUserPointer( + Os, + UserStr, + length, + (gctPOINTER *)&ptr + )); + + gckOS_MemCopy(_dumpStorage, ptr, length); + gckOS_UnmapUserPointer(Os, UserStr, length, ptr); + } + + _dumpStorage[length] = '\0'; + gcmkDUMP_STRING(Os, _dumpStorage); + + UserStr = (gctUINT8_PTR)UserStr + length; + offset += length; + } + + gcmkDUMP_STRING(Os, "\n"); + +OnError: + gcmkMUTEX_UNLOCK(_dumpMutex); +} + +static void +_DumpDataBuffer( + IN gckOS Os, + IN gceDUMP_BUFFER_TYPE Type, + IN gctPOINTER Data, + IN gctUINT64 Address, + IN gctSIZE_T Size + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctSIZE_T offset = 0; + gctSIZE_T length = 0; + gctBOOL needCopy = gcvTRUE; + gctCONST_STRING dumpTag; + char buffer[256]; + const gctSIZE_T maxLength = gcmSIZEOF(_dumpStorage); + + switch (Type) + { + case gcvDUMP_BUFFER_VERIFY: + dumpTag = "verify"; + break; + case gcvDUMP_BUFFER_PHYSICAL_MEMORY: + dumpTag = "physical"; + break; + default: + dumpTag = "memory"; + break; + } + + if (Type <= gcvDUMP_BUFFER_USER_TYPE_LAST) + { + gcmkVERIFY_OK(gckOS_QueryNeedCopy(Os, 0, &needCopy)); + } + + gcmkMUTEX_LOCK(_dumpMutex); + + /* Form and print the opening string. */ + if (Type == gcvDUMP_BUFFER_PHYSICAL_MEMORY) + { + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, + "@[%s 0x%010llX 0x%08X\n", + dumpTag, (unsigned long long)Address, (gctUINT32)Size); + } + else + { + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, + "@[%s 0x%08X 0x%08X\n", + dumpTag, (gctUINT32)Address, (gctUINT32)Size); + } + + gcmkDUMP_STRING(Os, buffer); + + + while (offset < Size) + { + gctPOINTER data = gcvNULL; + gctUINT32_PTR ptr; + gctUINT8_PTR bytePtr; + gctSIZE_T count, tailByteCount; + + length = maxLength < (Size - offset) ? maxLength : (Size - offset); + count = length / 4; + tailByteCount = length % 4; + + ptr = (gctUINT32_PTR)Data; + + if (Type <= gcvDUMP_BUFFER_USER_TYPE_LAST) + { + /* Copy or map from user. */ + if (needCopy) + { + gcmkONERROR(gckOS_CopyFromUserData( + Os, + _dumpStorage, + Data, + length + )); + + ptr = (gctUINT32_PTR)_dumpStorage; + } + else + { + gcmkONERROR(gckOS_MapUserPointer( + Os, + Data, + length, + (gctPOINTER *)&data + )); + + ptr = (gctUINT32_PTR)data; + } + } + + while (count >= 4) + { + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, + " 0x%08X 0x%08X 0x%08X 0x%08X\n", + ptr[0], ptr[1], ptr[2], ptr[3]); + + ptr += 4; + count -= 4; + + gcmkDUMP_STRING(Os, buffer); + } + + switch (count) + { + case 3: + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, + " 0x%08X 0x%08X 0x%08X", + ptr[0], ptr[1], ptr[2]); + break; + case 2: + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, + " 0x%08X 0x%08X", + ptr[0], ptr[1]); + break; + case 1: + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, " 0x%08X", ptr[0]); + break; + } + + if (count > 0) + { + gcmkDUMP_STRING(Os, buffer); + } + + bytePtr = (gctUINT8_PTR)(ptr + count); + + if (!count && tailByteCount) + { + /* There is an extra space for the new line. */ + gcmkDUMP_STRING(Os, " "); + } + + switch (tailByteCount) + { + case 3: + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, + " 0x00%02X%02X%02X", + bytePtr[2], bytePtr[1], bytePtr[0]); + break; + case 2: + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, + " 0x0000%02X%02X", + bytePtr[1], bytePtr[0]); + break; + case 1: + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, + " 0x000000%02X", bytePtr[0]); + break; + } + + if (tailByteCount) + { + gcmkDUMP_STRING(Os, buffer); + } + + if (count || tailByteCount) + { + gcmkDUMP_STRING(Os, "\n"); + } + + if (Type <= gcvDUMP_BUFFER_USER_TYPE_LAST && !needCopy) + { + gckOS_UnmapUserPointer(Os, Data, length, data); + } + /* advance to next batch. */ + Data = (gctUINT8_PTR)Data + length; + offset += length; + } + +OnError: + gcmkSPRINTF(buffer, gcmSIZEOF(buffer) - 1, "] -- %s\n", dumpTag); + gcmkDUMP_STRING(Os, buffer); + + gcmkMUTEX_UNLOCK(_dumpMutex); +} + +/******************************************************************************* +** +** gckOS_DumpBuffer +** +** Print the contents of the specified buffer. +** +** INPUT: +** +** gckOS Os +** Pointer to gckOS object. +** +** gceDUMP_BUFFER_TYPE Type +** Buffer type. +** +** gctPOINTER Buffer +** Pointer to the buffer to print. +** +** gctUINT64 Address +** Address. +** +** gctUINT Size +** Size of the buffer. +** +** +** OUTPUT: +** +** Nothing. +*/ +void +gckOS_DumpBuffer( + IN gckOS Os, + IN gceDUMP_BUFFER_TYPE Type, + IN gctPOINTER Buffer, + IN gctUINT64 Address, + IN gctSIZE_T Size + ) +{ + if (!Buffer) + { + return; + } + + /* memory dump below. */ + if (Type >= gcvDUMP_BUFFER_TYPE_COUNT) + { + gcmkPRINT("#[ERROR: invalid buffer type]\n"); + return; + } + + if (Type == gcvDUMP_BUFFER_USER_STRING) + { + _DumpUserString(Os, Buffer, Size); + } + else + { + _DumpDataBuffer(Os, Type, Buffer, Address, Size); + } +} + + +/******************************************************************************* +***** Binary Trace ************************************************************* +*******************************************************************************/ +
diff --git a/hal/kernel/gc_hal_kernel_event.c b/hal/kernel/gc_hal_kernel_event.c new file mode 100644 index 0000000..9f12fea --- /dev/null +++ b/hal/kernel/gc_hal_kernel_event.c
@@ -0,0 +1,2392 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" +#include "gc_hal_kernel_buffer.h" + +#ifdef __QNXNTO__ +#include "gc_hal_kernel_qnx.h" +#endif + +#define _GC_OBJ_ZONE gcvZONE_EVENT + +#define gcdEVENT_ALLOCATION_COUNT (4096 / gcmSIZEOF(gcsHAL_INTERFACE)) +#define gcdEVENT_MIN_THRESHOLD 4 + +/******************************************************************************\ +********************************* Support Code ********************************* +\******************************************************************************/ + +static gcmINLINE gceSTATUS +gckEVENT_AllocateQueue( + IN gckEVENT Event, + OUT gcsEVENT_QUEUE_PTR * Queue + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Queue != gcvNULL); + + /* Do we have free queues? */ + if (Event->freeList == gcvNULL) + { + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + + /* Move one free queue from the free list. */ + * Queue = Event->freeList; + Event->freeList = Event->freeList->next; + + /* Success. */ + gcmkFOOTER_ARG("*Queue=0x%x", gcmOPT_POINTER(Queue)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckEVENT_FreeQueue( + IN gckEVENT Event, + OUT gcsEVENT_QUEUE_PTR Queue + ) +{ + gceSTATUS status = gcvSTATUS_OK; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Queue != gcvNULL); + + /* Move one free queue from the free list. */ + Queue->next = Event->freeList; + Event->freeList = Queue; + + /* Success. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckEVENT_FreeRecord( + IN gckEVENT Event, + IN gcsEVENT_PTR Record + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Record != gcvNULL); + + /* Acquire the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->freeEventMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + /* Push the record on the free list. */ + Record->next = Event->freeEventList; + Event->freeEventList = Record; + Event->freeEventCount += 1; + + /* Release the mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return gcvSTATUS_OK; +} + +static gceSTATUS +gckEVENT_IsEmpty( + IN gckEVENT Event, + OUT gctBOOL_PTR IsEmpty + ) +{ + gceSTATUS status; + gctSIZE_T i; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(IsEmpty != gcvNULL); + + /* Assume the event queue is empty. */ + *IsEmpty = gcvTRUE; + + /* Walk the event queue. */ + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + /* Check whether this event is in use. */ + if (Event->queues[i].head != gcvNULL) + { + /* The event is in use, hence the queue is not empty. */ + *IsEmpty = gcvFALSE; + break; + } + } + + /* Try acquiring the mutex. */ + status = gckOS_AcquireMutex(Event->os, Event->eventQueueMutex, 0); + if (status == gcvSTATUS_TIMEOUT) + { + /* Timeout - queue is no longer empty. */ + *IsEmpty = gcvFALSE; + } + else + { + /* Bail out on error. */ + gcmkONERROR(status); + + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + } + + /* Success. */ + gcmkFOOTER_ARG("*IsEmpty=%d", gcmOPT_VALUE(IsEmpty)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_TryToIdleGPU( + IN gckEVENT Event + ) +{ + gceSTATUS status; + gctBOOL empty = gcvFALSE, idle = gcvFALSE; + gctBOOL powerLocked = gcvFALSE; + gckHARDWARE hardware; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + /* Grab gckHARDWARE object. */ + hardware = Event->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Check whether the event queue is empty. */ + gcmkONERROR(gckEVENT_IsEmpty(Event, &empty)); + + if (empty) + { + status = gckOS_AcquireMutex(hardware->os, hardware->powerMutex, 0); + if (status == gcvSTATUS_TIMEOUT) + { + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + powerLocked = gcvTRUE; + + /* Query whether the hardware is idle. */ + gcmkONERROR(gckHARDWARE_QueryIdle(Event->kernel->hardware, &idle)); + + gcmkONERROR(gckOS_ReleaseMutex(hardware->os, hardware->powerMutex)); + powerLocked = gcvFALSE; + + if (idle) + { + /* Inform the system of idle GPU. */ + gcmkONERROR(gckOS_Broadcast(Event->os, + Event->kernel->hardware, + gcvBROADCAST_GPU_IDLE)); + } + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (powerLocked) + { + gcmkONERROR(gckOS_ReleaseMutex(hardware->os, hardware->powerMutex)); + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +__RemoveRecordFromProcessDB( + IN gckEVENT Event, + IN gcsEVENT_PTR Record + ) +{ + gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record); + gcmkVERIFY_ARGUMENT(Record != gcvNULL); + + switch (Record->info.command) + { + case gcvHAL_UNLOCK_VIDEO_MEMORY: + gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB( + Event->kernel, + Record->processID, + gcvDB_VIDEO_MEMORY_LOCKED, + gcmUINT64_TO_PTR(Record->info.u.UnlockVideoMemory.node))); + break; + + default: + break; + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +static gceSTATUS +_ReleaseVideoMemoryHandle( + IN gckKERNEL Kernel, + IN OUT gcsEVENT_PTR Record, + IN OUT gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject; + gctUINT32 handle; + + switch(Interface->command) + { + case gcvHAL_UNLOCK_VIDEO_MEMORY: + handle = (gctUINT32)Interface->u.UnlockVideoMemory.node; + + gcmkONERROR(gckVIDMEM_HANDLE_Lookup( + Kernel, Record->processID, handle, &nodeObject)); + + Record->info.u.UnlockVideoMemory.node = gcmPTR_TO_UINT64(nodeObject); + + gckVIDMEM_HANDLE_Dereference(Kernel, Record->processID, handle); + break; + + default: + break; + } + + return gcvSTATUS_OK; +OnError: + return status; +} + +/******************************************************************************* +** +** _QueryFlush +** +** Check the type of surfaces which will be released by current event and +** determine the cache needed to flush. +** +*/ +static gceSTATUS +_QueryFlush( + IN gckEVENT Event, + IN gcsEVENT_PTR Record, + OUT gceKERNEL_FLUSH *Flush + ) +{ + gceKERNEL_FLUSH flush = 0; + gckVIDMEM_NODE nodeObject; + + gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record); + gcmkVERIFY_ARGUMENT(Record != gcvNULL); + + while (Record != gcvNULL) + { + switch (Record->info.command) + { + case gcvHAL_UNLOCK_VIDEO_MEMORY: + nodeObject = gcmUINT64_TO_PTR(Record->info.u.UnlockVideoMemory.node); + + switch (nodeObject->type) + { + case gcvVIDMEM_TYPE_TILE_STATUS: + flush |= gcvFLUSH_TILE_STATUS; + break; + case gcvVIDMEM_TYPE_COLOR_BUFFER: + flush |= gcvFLUSH_COLOR; + break; + case gcvVIDMEM_TYPE_DEPTH_BUFFER: + flush |= gcvFLUSH_DEPTH; + break; + case gcvVIDMEM_TYPE_TEXTURE: + flush |= gcvFLUSH_TEXTURE; + break; + case gcvVIDMEM_TYPE_ICACHE: + flush |= gcvFLUSH_ICACHE; + break; + case gcvVIDMEM_TYPE_TXDESC: + flush |= gcvFLUSH_TXDESC; + break; + case gcvVIDMEM_TYPE_FENCE: + flush |= gcvFLUSH_FENCE; + break; + case gcvVIDMEM_TYPE_VERTEX_BUFFER: + flush |= gcvFLUSH_VERTEX; + break; + case gcvVIDMEM_TYPE_TFBHEADER: + flush |= gcvFLUSH_TFBHEADER; + break; + case gcvVIDMEM_TYPE_GENERIC: + flush = gcvFLUSH_ALL; + goto Out; + default: + break; + } + break; + default: + break; + } + + Record = Record->next; + } + +Out: + *Flush = flush; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +void +_SubmitTimerFunction( + gctPOINTER Data + ) +{ + gckEVENT event = (gckEVENT)Data; + gcmkVERIFY_OK(gckEVENT_Submit(event, gcvTRUE, gcvFALSE)); +} + +/******************************************************************************\ +******************************* gckEVENT API Code ******************************* +\******************************************************************************/ + +/******************************************************************************* +** +** gckEVENT_Construct +** +** Construct a new gckEVENT object. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** OUTPUT: +** +** gckEVENT * Event +** Pointer to a variable that receives the gckEVENT object pointer. +*/ +gceSTATUS +gckEVENT_Construct( + IN gckKERNEL Kernel, + IN gckCOMMAND Command, + OUT gckEVENT * Event + ) +{ + gckOS os; + gceSTATUS status; + gckEVENT eventObj = gcvNULL; + int i; + gcsEVENT_PTR record; + gctPOINTER pointer = gcvNULL; + + gcmkHEADER_ARG("Kernel=0x%x", Kernel); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Event != gcvNULL); + + /* Extract the pointer to the gckOS object. */ + os = Kernel->os; + gcmkVERIFY_OBJECT(os, gcvOBJ_OS); + + /* Allocate the gckEVENT object. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckEVENT), &pointer)); + + eventObj = pointer; + + /* Reset the object. */ + gcmkVERIFY_OK(gckOS_ZeroMemory(eventObj, gcmSIZEOF(struct _gckEVENT))); + + /* Initialize the gckEVENT object. */ + eventObj->object.type = gcvOBJ_EVENT; + eventObj->kernel = Kernel; + eventObj->os = os; + eventObj->command = Command; + + /* Create the mutexes. */ + gcmkONERROR(gckOS_CreateMutex(os, &eventObj->eventQueueMutex)); + gcmkONERROR(gckOS_CreateMutex(os, &eventObj->freeEventMutex)); + gcmkONERROR(gckOS_CreateMutex(os, &eventObj->eventListMutex)); + + /* Create a bunch of event reccords. */ + for (i = 0; i < gcdEVENT_ALLOCATION_COUNT; i += 1) + { + /* Allocate an event record. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsEVENT), &pointer)); + + record = pointer; + + /* Push it on the free list. */ + record->next = eventObj->freeEventList; + eventObj->freeEventList = record; + eventObj->freeEventCount += 1; + } + + /* Initialize the free list of event queues. */ + for (i = 0; i < gcdREPO_LIST_COUNT; i += 1) + { + eventObj->repoList[i].next = eventObj->freeList; + eventObj->freeList = &eventObj->repoList[i]; + } + + eventObj->freeQueueCount = gcmCOUNTOF(eventObj->queues); + + gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->pending)); + + gcmkVERIFY_OK(gckOS_CreateTimer(os, + _SubmitTimerFunction, + (gctPOINTER)eventObj, + &eventObj->submitTimer)); + +#if gcdINTERRUPT_STATISTIC + gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->interruptCount)); + gcmkONERROR(gckOS_AtomSet(os,eventObj->interruptCount, 0)); +#endif + + eventObj->notifyState = -1; + + /* Return pointer to the gckEVENT object. */ + *Event = eventObj; + + /* Success. */ + gcmkFOOTER_ARG("*Event=0x%x", *Event); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (eventObj != gcvNULL) + { + if (eventObj->eventQueueMutex != gcvNULL) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->eventQueueMutex)); + } + + if (eventObj->freeEventMutex != gcvNULL) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->freeEventMutex)); + } + + if (eventObj->eventListMutex != gcvNULL) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->eventListMutex)); + } + + while (eventObj->freeEventList != gcvNULL) + { + record = eventObj->freeEventList; + eventObj->freeEventList = record->next; + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, record)); + } + + if (eventObj->pending != gcvNULL) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->pending)); + } + +#if gcdINTERRUPT_STATISTIC + if (eventObj->interruptCount) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->interruptCount)); + } +#endif + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, eventObj)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Destroy +** +** Destroy an gckEVENT object. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Destroy( + IN gckEVENT Event + ) +{ + gcsEVENT_PTR record; + gcsEVENT_QUEUE_PTR queue; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + if (Event->submitTimer != gcvNULL) + { + gcmkVERIFY_OK(gckOS_StopTimer(Event->os, Event->submitTimer)); + gcmkVERIFY_OK(gckOS_DestroyTimer(Event->os, Event->submitTimer)); + } + + /* Delete the queue mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->eventQueueMutex)); + + /* Free all free events. */ + while (Event->freeEventList != gcvNULL) + { + record = Event->freeEventList; + Event->freeEventList = record->next; + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, record)); + } + + /* Delete the free mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->freeEventMutex)); + + /* Free all pending queues. */ + while (Event->queueHead != gcvNULL) + { + /* Get the current queue. */ + queue = Event->queueHead; + + /* Free all pending events. */ + while (queue->head != gcvNULL) + { + record = queue->head; + queue->head = record->next; + + gcmkTRACE_ZONE_N( + gcvLEVEL_WARNING, gcvZONE_EVENT, + gcmSIZEOF(record) + gcmSIZEOF(queue->source), + "Event record 0x%x is still pending for %d.", + record, queue->source + ); + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, record)); + } + + /* Remove the top queue from the list. */ + if (Event->queueHead == Event->queueTail) + { + Event->queueHead = + Event->queueTail = gcvNULL; + } + else + { + Event->queueHead = Event->queueHead->next; + } + + /* Free the queue. */ + gcmkVERIFY_OK(gckEVENT_FreeQueue(Event, queue)); + } + + /* Delete the list mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->eventListMutex)); + + gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->pending)); + +#if gcdINTERRUPT_STATISTIC + gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->interruptCount)); +#endif + + /* Mark the gckEVENT object as unknown. */ + Event->object.type = gcvOBJ_UNKNOWN; + + /* Free the gckEVENT object. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, Event)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckEVENT_GetEvent +** +** Reserve the next available hardware event. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctBOOL Wait +** Set to gcvTRUE to force the function to wait if no events are +** immediately available. +** +** gceKERNEL_WHERE Source +** Source of the event. +** +** OUTPUT: +** +** gctUINT8 * EventID +** Reserved event ID. +*/ +#define gcdINVALID_EVENT_PTR ((gcsEVENT_PTR)gcvMAXUINTPTR_T) + +gceSTATUS +gckEVENT_GetEvent( + IN gckEVENT Event, + IN gctBOOL Wait, + OUT gctUINT8 * EventID, + IN gceKERNEL_WHERE Source + ) +{ + gctINT i, id; + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Event=0x%x Source=%d", Event, Source); + + while (gcvTRUE) + { + /* Grab the queue mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->eventQueueMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + /* Walk through all events. */ + id = Event->lastID; + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + gctINT nextID = id + 1; + + if (nextID == gcmCOUNTOF(Event->queues)) + { + nextID = 0; + } + + if (Event->queues[id].head == gcvNULL) + { + *EventID = (gctUINT8) id; + + Event->lastID = (gctUINT8) nextID; + + /* Save time stamp of event. */ + Event->queues[id].head = gcdINVALID_EVENT_PTR; + Event->queues[id].stamp = ++(Event->stamp); + Event->queues[id].source = Source; + + /* Decrease the number of free events. */ + --Event->freeQueueCount; + +#if gcdDYNAMIC_SPEED + if (Event->freeQueueCount <= gcdDYNAMIC_EVENT_THRESHOLD) + { + gcmkONERROR(gckOS_BroadcastHurry( + Event->os, + Event->kernel->hardware, + gcdDYNAMIC_EVENT_THRESHOLD - Event->freeQueueCount)); + } +#endif + + /* Release the queue mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, + Event->eventQueueMutex)); + + /* Success. */ + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_EVENT, + gcmSIZEOF(id), + "Using id=%d", + id + ); + + gcmkFOOTER_ARG("*EventID=%u", *EventID); + return gcvSTATUS_OK; + } + + id = nextID; + } + +#if gcdDYNAMIC_SPEED + /* No free events, speed up the GPU right now! */ + gcmkONERROR(gckOS_BroadcastHurry(Event->os, + Event->kernel->hardware, + gcdDYNAMIC_EVENT_THRESHOLD)); +#endif + + /* Release the queue mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + + /* Fail if wait is not requested. */ + if (!Wait) + { + /* Out of resources. */ + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + + /* Delay a while. */ + gcmkONERROR(gckOS_Delay(Event->os, 1)); + } + +OnError: + if (acquired) + { + /* Release the queue mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_AllocateRecord +** +** Allocate a record for the new event. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctBOOL AllocateAllowed +** State for allocation if out of free events. +** +** OUTPUT: +** +** gcsEVENT_PTR * Record +** Allocated event record. +*/ +static gcmINLINE gceSTATUS +gckEVENT_AllocateRecord( + IN gckEVENT Event, + IN gctBOOL AllocateAllowed, + OUT gcsEVENT_PTR * Record + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gctINT i; + gcsEVENT_PTR record; + gctPOINTER pointer = gcvNULL; + + gcmkHEADER_ARG("Event=0x%x AllocateAllowed=%d", Event, AllocateAllowed); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Record != gcvNULL); + + /* Acquire the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, Event->freeEventMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Test if we are below the allocation threshold. */ + if ( (AllocateAllowed && (Event->freeEventCount < gcdEVENT_MIN_THRESHOLD)) || + (Event->freeEventCount == 0) ) + { + /* Allocate a bunch of records. */ + for (i = 0; i < gcdEVENT_ALLOCATION_COUNT; i += 1) + { + /* Allocate an event record. */ + gcmkONERROR(gckOS_Allocate(Event->os, + gcmSIZEOF(gcsEVENT), + &pointer)); + + record = pointer; + + /* Push it on the free list. */ + record->next = Event->freeEventList; + Event->freeEventList = record; + Event->freeEventCount += 1; + } + } + + *Record = Event->freeEventList; + Event->freeEventList = Event->freeEventList->next; + Event->freeEventCount -= 1; + + /* Release the mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex)); + + /* Success. */ + gcmkFOOTER_ARG("*Record=0x%x", gcmOPT_POINTER(Record)); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_AddList +** +** Add a new event to the list of events. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gcsHAL_INTERFACE_PTR Interface +** Pointer to the interface for the event to be added. +** +** gceKERNEL_WHERE FromWhere +** Place in the pipe where the event needs to be generated. +** +** gctBOOL AllocateAllowed +** State for allocation if out of free events. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_AddList( + IN gckEVENT Event, + IN gcsHAL_INTERFACE_PTR Interface, + IN gceKERNEL_WHERE FromWhere, + IN gctBOOL AllocateAllowed, + IN gctBOOL FromKernel + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gcsEVENT_PTR record = gcvNULL; + gcsEVENT_QUEUE_PTR queue; + + gcmkHEADER_ARG("Event=0x%x Interface=0x%x", + Event, Interface); + + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, _GC_OBJ_ZONE, + "FromWhere=%d AllocateAllowed=%d", + FromWhere, AllocateAllowed); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Interface != gcvNULL); + + /* Verify the event command. */ + gcmkASSERT + ( (Interface->command == gcvHAL_WRITE_DATA) + || (Interface->command == gcvHAL_UNLOCK_VIDEO_MEMORY) + || (Interface->command == gcvHAL_SIGNAL) + || (Interface->command == gcvHAL_TIMESTAMP) + || (Interface->command == gcvHAL_COMMIT_DONE) + || (Interface->command == gcvHAL_DESTROY_MMU) + ); + + /* Validate the source. */ + if ((FromWhere != gcvKERNEL_COMMAND) && (FromWhere != gcvKERNEL_PIXEL)) + { + /* Invalid argument. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + /* Allocate a free record. */ + gcmkONERROR(gckEVENT_AllocateRecord(Event, AllocateAllowed, &record)); + + /* Termninate the record. */ + record->next = gcvNULL; + + /* Record the committer. */ + record->fromKernel = FromKernel; + + /* Copy the event interface into the record. */ + gckOS_MemCopy(&record->info, Interface, gcmSIZEOF(record->info)); + + /* Get process ID. */ + gcmkONERROR(gckOS_GetProcessID(&record->processID)); + + if (FromKernel == gcvFALSE) + { + gcmkONERROR(__RemoveRecordFromProcessDB(Event, record)); + + /* Handle is belonged to current process, it must be released now. */ + status = _ReleaseVideoMemoryHandle(Event->kernel, record, Interface); + + if (gcmIS_ERROR(status)) + { + /* Ingore error because there are other events in the queue. */ + status = gcvSTATUS_OK; + goto OnError; + } + } + +#ifdef __QNXNTO__ + record->kernel = Event->kernel; +#endif + + /* Acquire the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, Event->eventListMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Do we need to allocate a new queue? */ + if ((Event->queueTail == gcvNULL) || (Event->queueTail->source < FromWhere)) + { + /* Allocate a new queue. */ + gcmkONERROR(gckEVENT_AllocateQueue(Event, &queue)); + + /* Initialize the queue. */ + queue->source = FromWhere; + queue->head = gcvNULL; + queue->next = gcvNULL; + + /* Attach it to the list of allocated queues. */ + if (Event->queueTail == gcvNULL) + { + Event->queueHead = + Event->queueTail = queue; + } + else + { + Event->queueTail->next = queue; + Event->queueTail = queue; + } + } + else + { + queue = Event->queueTail; + } + + /* Attach the record to the queue. */ + if (queue->head == gcvNULL) + { + queue->head = record; + queue->tail = record; + } + else + { + queue->tail->next = record; + queue->tail = record; + } + + /* Release the mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + } + + if (record != gcvNULL) + { + gcmkVERIFY_OK(gckEVENT_FreeRecord(Event, record)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Unlock +** +** Schedule an event to unlock virtual memory. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gceKERNEL_WHERE FromWhere +** Place in the pipe where the event needs to be generated. +** +** gcuVIDMEM_NODE_PTR Node +** Pointer to a gcuVIDMEM_NODE union that specifies the virtual memory +** to unlock. +** +** gceVIDMEM_TYPE Type +** Video memory allocation type to unlock. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Unlock( + IN gckEVENT Event, + IN gceKERNEL_WHERE FromWhere, + IN gctPOINTER Node + ) +{ + gceSTATUS status; + gcsHAL_INTERFACE iface; + + gcmkHEADER_ARG("Event=0x%x FromWhere=%d Node=0x%x", + Event, FromWhere, Node); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Node != gcvNULL); + + /* Mark the event as an unlock. */ + iface.command = gcvHAL_UNLOCK_VIDEO_MEMORY; + iface.u.UnlockVideoMemory.node = gcmPTR_TO_UINT64(Node); + iface.u.UnlockVideoMemory.asynchroneous = 0; + + /* Append it to the queue. */ + gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Signal +** +** Schedule an event to trigger a signal. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctSIGNAL Signal +** Pointer to the signal to trigger. +** +** gceKERNEL_WHERE FromWhere +** Place in the pipe where the event needs to be generated. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Signal( + IN gckEVENT Event, + IN gctSIGNAL Signal, + IN gceKERNEL_WHERE FromWhere + ) +{ + gceSTATUS status; + gcsHAL_INTERFACE iface; + + gcmkHEADER_ARG("Event=0x%x Signal=0x%x FromWhere=%d", + Event, Signal, FromWhere); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Signal != gcvNULL); + + /* Mark the event as a signal. */ + iface.command = gcvHAL_SIGNAL; + iface.u.Signal.signal = gcmPTR_TO_UINT64(Signal); + iface.u.Signal.auxSignal = 0; + iface.u.Signal.process = 0; + +#ifdef __QNXNTO__ + iface.u.Signal.coid = 0; + iface.u.Signal.rcvid = 0; + + gcmkONERROR(gckOS_SignalPending(Event->os, Signal)); +#endif + + /* Append it to the queue. */ + gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Submit +** +** Submit the current event queue to the GPU. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctBOOL Wait +** Submit requires one vacant event; if Wait is set to not zero, +** and there are no vacant events at this time, the function will +** wait until an event becomes vacant so that submission of the +** queue is successful. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Submit( + IN gckEVENT Event, + IN gctBOOL Wait, + IN gctBOOL FromPower + ) +{ + gceSTATUS status; + gctUINT8 id = 0xFF; + gcsEVENT_QUEUE_PTR queue; + gctBOOL acquired = gcvFALSE; + gckCOMMAND command = gcvNULL; + gctBOOL commitEntered = gcvFALSE; + gctUINT32 bytes; + gctPOINTER buffer; + gctUINT32 executeBytes; + gctUINT32 flushBytes; + +#if gcdINTERRUPT_STATISTIC + gctINT32 oldValue; +#endif + +#if gcdSECURITY + gctPOINTER reservedBuffer; +#endif + + gckHARDWARE hardware; + + gceKERNEL_FLUSH flush = gcvFALSE; + gctUINT64 commitStamp; + + gcmkHEADER_ARG("Event=0x%x Wait=%d", Event, Wait); + + /* Get gckCOMMAND object. */ + command = Event->command; + hardware = Event->kernel->hardware; + + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + gckOS_GetTicks(&Event->lastCommitStamp); + + /* Are there event queues? */ + if (Event->queueHead != gcvNULL) + { + /* Acquire the command queue. */ + gcmkONERROR(gckCOMMAND_EnterCommit(command, FromPower)); + commitEntered = gcvTRUE; + + /* Get current commit stamp. */ + commitStamp = command->commitStamp; + + if (commitStamp) + { + commitStamp -= 1; + } + + /* Process all queues. */ + while (Event->queueHead != gcvNULL) + { + /* Acquire the list mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->eventListMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + /* Get the current queue. */ + queue = Event->queueHead; + + /* Allocate an event ID. */ + gcmkONERROR(gckEVENT_GetEvent(Event, Wait, &id, queue->source)); + + /* Copy event list to event ID queue. */ + Event->queues[id].head = queue->head; + + /* Update current commit stamp. */ + Event->queues[id].commitStamp = commitStamp; + + /* Remove the top queue from the list. */ + if (Event->queueHead == Event->queueTail) + { + Event->queueHead = gcvNULL; + Event->queueTail = gcvNULL; + } + else + { + Event->queueHead = Event->queueHead->next; + } + + /* Free the queue. */ + gcmkONERROR(gckEVENT_FreeQueue(Event, queue)); + + /* Release the list mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + acquired = gcvFALSE; + + + if (command->feType == gcvHW_FE_WAIT_LINK) + { + /* Determine cache needed to flush. */ + gcmkVERIFY_OK(_QueryFlush(Event, Event->queues[id].head, &flush)); + + /* Get the size of the hardware event. */ + gcmkONERROR(gckWLFE_Event( + hardware, + gcvNULL, + id, + Event->queues[id].source, + &bytes + )); + + /* Get the size of flush command. */ + gcmkONERROR(gckHARDWARE_Flush( + hardware, + flush, + gcvNULL, + &flushBytes + )); + + bytes += flushBytes; + } + else if (command->feType == gcvHW_FE_ASYNC) + { + /* Get the size of the hardware event. */ + gcmkONERROR(gckASYNC_FE_Event( + hardware, + gcvNULL, + id, + Event->queues[id].source, + &bytes + )); + } + else + { + /* Get the size of the hardware event. */ + gcmkONERROR(gckMCFE_Event( + hardware, + gcvNULL, + id, + Event->queues[id].source, + &bytes + )); + } + + /* Total bytes need to execute. */ + executeBytes = bytes; + + /* Reserve space in the command queue. */ + gcmkONERROR(gckCOMMAND_Reserve(command, bytes, &buffer, &bytes)); +#if gcdSECURITY + reservedBuffer = buffer; +#endif + +#if gcdINTERRUPT_STATISTIC + gcmkVERIFY_OK(gckOS_AtomIncrement( + Event->os, + Event->interruptCount, + &oldValue + )); +#endif + + if (command->feType == gcvHW_FE_WAIT_LINK) + { + /* Set the flush in the command queue. */ + gcmkONERROR(gckHARDWARE_Flush( + hardware, + flush, + buffer, + &flushBytes + )); + + /* Advance to next command. */ + buffer = (gctUINT8_PTR)buffer + flushBytes; + + /* Set the hardware event in the command queue. */ + gcmkONERROR(gckWLFE_Event( + hardware, + buffer, + id, + Event->queues[id].source, + &bytes + )); + +#if gcdSECURITY + gckKERNEL_SecurityExecute( + Event->kernel, + reservedBuffer, + executeBytes + ); +#else + /* Execute the hardware event. */ + gcmkONERROR(gckCOMMAND_Execute(command, executeBytes)); +#endif + } + else if (command->feType == gcvHW_FE_ASYNC) + { + /* Set the hardware event in the command queue. */ + gcmkONERROR(gckASYNC_FE_Event( + hardware, + buffer, + id, + Event->queues[id].source, + &bytes + )); + + /* Execute the hardware event. */ + gcmkONERROR(gckCOMMAND_ExecuteAsync(command, executeBytes)); + } + else + { + /* Set the hardware event in the command queue. */ + gcmkONERROR(gckMCFE_Event( + hardware, + buffer, + id, + Event->queues[id].source, + &bytes + )); + + /* Execute the hardware event. */ + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(command, 0, 0, executeBytes)); + } + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* Notify immediately on infinite hardware. */ + gcmkONERROR(gckEVENT_Interrupt(Event, 1 << id)); + + gcmkONERROR(gckEVENT_Notify(Event, 0)); +#endif + } + + /* Release the command queue. */ + gcmkONERROR(gckCOMMAND_ExitCommit(command, FromPower)); + +#if !gcdNULL_DRIVER + if (!FromPower) + { + gcmkVERIFY_OK(_TryToIdleGPU(Event)); + } +#endif + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Need to unroll the mutex acquire. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + } + + if (commitEntered) + { + /* Release the command queue mutex. */ + gcmkVERIFY_OK(gckCOMMAND_ExitCommit(command, FromPower)); + } + + if (id != 0xFF) + { + /* Need to unroll the event allocation. */ + Event->queues[id].head = gcvNULL; + } + + if (status == gcvSTATUS_GPU_NOT_RESPONDING) + { + /* Broadcast GPU stuck. */ + status = gckOS_Broadcast(Event->os, + Event->kernel->hardware, + gcvBROADCAST_GPU_STUCK); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Commit +** +** Commit an event queue from the user. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gcsQUEUE_PTR Queue +** User event queue. +** +** gctBOOL Forced +** Force fire a event. There won't be interrupt if there's no events + queued. Force a event by append a dummy one if this parameter is on. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Commit( + IN gckEVENT Event, + IN gcsQUEUE_PTR Queue, + IN gctBOOL Forced + ) +{ + gceSTATUS status; + gcsQUEUE_PTR record = gcvNULL, next; + gctUINT32 processID; + gctBOOL needCopy = gcvFALSE; + gctPOINTER pointer = gcvNULL; + + gcmkHEADER_ARG("Event=0x%x Queue=0x%x", Event, Queue); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + /* Get the current process ID. */ + gcmkONERROR(gckOS_GetProcessID(&processID)); + + /* Query if we need to copy the client data. */ + gcmkONERROR(gckOS_QueryNeedCopy(Event->os, processID, &needCopy)); + + /* Loop while there are records in the queue. */ + while (Queue != gcvNULL) + { + gcsQUEUE queue; + + if (needCopy) + { + /* Point to stack record. */ + record = &queue; + + /* Copy the data from the client. */ + gcmkONERROR(gckOS_CopyFromUserData(Event->os, + record, + Queue, + gcmSIZEOF(gcsQUEUE))); + } + else + { + + /* Map record into kernel memory. */ + gcmkONERROR(gckOS_MapUserPointer(Event->os, + Queue, + gcmSIZEOF(gcsQUEUE), + &pointer)); + + record = pointer; + } + + /* Append event record to event queue. */ + gcmkONERROR( + gckEVENT_AddList(Event, &record->iface, gcvKERNEL_PIXEL, gcvTRUE, gcvFALSE)); + + /* Next record in the queue. */ + next = gcmUINT64_TO_PTR(record->next); + + if (!needCopy) + { + /* Unmap record from kernel memory. */ + gcmkONERROR( + gckOS_UnmapUserPointer(Event->os, + Queue, + gcmSIZEOF(gcsQUEUE), + (gctPOINTER *) record)); + record = gcvNULL; + } + + Queue = next; + } + + if (Forced && Event->queueHead == gcvNULL) + { + gcsHAL_INTERFACE iface; + iface.command = gcvHAL_COMMIT_DONE; + + gcmkONERROR(gckEVENT_AddList(Event, &iface, gcvKERNEL_PIXEL, gcvFALSE, gcvTRUE)); + } + + /* Submit the event list. */ + gcmkONERROR(gckEVENT_Submit(Event, gcvTRUE, gcvFALSE)); + + /* Success */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (pointer) + { + /* Roll back. */ + gcmkVERIFY_OK(gckOS_UnmapUserPointer(Event->os, + Queue, + gcmSIZEOF(gcsQUEUE), + (gctPOINTER*)pointer)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Interrupt +** +** Called by the interrupt service routine to store the triggered interrupt +** mask to be later processed by gckEVENT_Notify. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctUINT32 Data +** Mask for the 32 interrupts. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Interrupt( + IN gckEVENT Event, + IN gctUINT32 Data + ) +{ + /* Combine current interrupt status with pending flags. */ + gckOS_AtomSetMask(Event->pending, Data); + +#if gcdINTERRUPT_STATISTIC + { + gctINT j = 0; + gctINT32 oldValue; + + for (j = 0; j < gcmCOUNTOF(Event->queues); j++) + { + if ((Data & (1 << j))) + { + gckOS_AtomDecrement(Event->os, + Event->interruptCount, + &oldValue); + } + } + } +#endif + + /* Success. */ + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckEVENT_Notify +** +** Process all triggered interrupts. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Notify( + IN gckEVENT Event, + IN gctUINT32 IDs + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctINT i; + gcsEVENT_QUEUE * queue; + gctUINT mask = 0; + gctBOOL acquired = gcvFALSE; + gctSIGNAL signal; + gctUINT pending = 0; + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gctINT eventNumber = 0; +#endif + gckVIDMEM_NODE nodeObject; + + gcmkHEADER_ARG("Event=0x%x IDs=0x%x", Event, IDs); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + gcmDEBUG_ONLY( + if (IDs != 0) + { + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if (Event->queues[i].head != gcvNULL) + { + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "Queue(%d): stamp=%llu source=%d", + i, + Event->queues[i].stamp, + Event->queues[i].source); + } + } + } + ); + + /* Begin of event handling. */ + Event->notifyState = 0; + + for (;;) + { + gcsEVENT_PTR record; + + /* Grab the mutex queue. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->eventQueueMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + gckOS_AtomGet(Event->os, Event->pending, (gctINT32_PTR)&pending); + + if (pending == 0) + { + /* Release the mutex queue. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + + /* No more pending interrupts - done. */ + break; + } + + if (pending & 0x80000000) + { + gcmkPRINT("AXI BUS ERROR"); + gckHARDWARE_DumpGPUState(Event->kernel->hardware); + pending &= 0x7FFFFFFF; + } + + if ((pending & 0x40000000) && Event->kernel->hardware->mmuVersion) + { +#if gcdUSE_MMU_EXCEPTION +#if gcdALLOC_ON_FAULT + status = gckHARDWARE_HandleFault(Event->kernel->hardware); +#endif + if (gcmIS_ERROR(status)) + { + /* Dump error is fault can't be handle. */ + gckHARDWARE_DumpMMUException(Event->kernel->hardware); + + gckHARDWARE_DumpGPUState(Event->kernel->hardware); + } +#endif + + pending &= 0xBFFFFFFF; + } + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_EVENT, + gcmSIZEOF(pending), + "Pending interrupts 0x%x", + pending + ); + + queue = gcvNULL; + + gcmDEBUG_ONLY( + if (IDs == 0) + { + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if (Event->queues[i].head != gcvNULL) + { + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "Queue(%d): stamp=%llu source=%d", + i, + Event->queues[i].stamp, + Event->queues[i].source); + } + } + } + ); + + /* Find the oldest pending interrupt. */ + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if ((Event->queues[i].head != gcvNULL) + && (pending & (1 << i)) + ) + { + if ((queue == gcvNULL) + || (Event->queues[i].stamp < queue->stamp) + ) + { + queue = &Event->queues[i]; + mask = 1 << i; +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + eventNumber = i; +#endif + } + } + } + + if (queue == gcvNULL) + { + gcmkTRACE_ZONE_N( + gcvLEVEL_ERROR, gcvZONE_EVENT, + gcmSIZEOF(pending), + "Interrupts 0x%x are not pending.", + pending + ); + + gckOS_AtomClearMask(Event->pending, pending); + + /* Release the mutex queue. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + break; + } + + /* Check whether there is a missed interrupt. */ + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if ((Event->queues[i].head != gcvNULL) + && (Event->queues[i].stamp < queue->stamp) + && (Event->queues[i].source <= queue->source) + ) + { + gcmkTRACE_N( + gcvLEVEL_ERROR, + gcmSIZEOF(i) + gcmSIZEOF(Event->queues[i].stamp), + "Event %d lost (stamp %llu)", + i, Event->queues[i].stamp + ); + + /* Use this event instead. */ + queue = &Event->queues[i]; + mask = 0; + } + } + + if (mask != 0) + { +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_EVENT, + gcmSIZEOF(eventNumber), + "Processing interrupt %d", + eventNumber + ); +#endif + } + + gckOS_AtomClearMask(Event->pending, mask); + + if (!gckHARDWARE_IsFeatureAvailable(Event->kernel->hardware, gcvFEATURE_FENCE_64BIT)) + { + /* Write out commit stamp.*/ + *(gctUINT64 *)(Event->kernel->command->fence->logical) = queue->commitStamp; + } + + /* Signal clients waiting for fence. */ + gcmkVERIFY_OK(gckFENCE_Signal( + Event->os, + Event->kernel->command->fence + )); + + /* Grab the event head. */ + record = queue->head; + + /* Now quickly clear its event list. */ + queue->head = gcvNULL; + + /* Increase the number of free events. */ + Event->freeQueueCount++; + + /* Release the mutex queue. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + + /* Walk all events for this interrupt. */ + while (record != gcvNULL) + { + gcsEVENT_PTR recordNext; +#ifndef __QNXNTO__ + gctPOINTER logical; +#endif + /* Grab next record. */ + recordNext = record->next; + +#ifdef __QNXNTO__ + /* + * Assign record->processID as the pid for this galcore thread. + * Used in the OS calls which do not take a pid. + */ + drv_thread_specific_key_assign(record->processID, 0); +#endif + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_EVENT, + gcmSIZEOF(record->info.command), + "Processing event type: %d", + record->info.command + ); + + switch (record->info.command) + { + case gcvHAL_WRITE_DATA: +#ifndef __QNXNTO__ + /* Convert physical into logical address. */ + gcmkERR_BREAK( + gckOS_MapPhysical(Event->os, + record->info.u.WriteData.address, + gcmSIZEOF(gctUINT32), + &logical)); + + /* Write data. */ + gcmkERR_BREAK( + gckOS_WriteMemory(Event->os, + logical, + record->info.u.WriteData.data)); + + /* Unmap the physical memory. */ + gcmkERR_BREAK( + gckOS_UnmapPhysical(Event->os, + logical, + gcmSIZEOF(gctUINT32))); +#else + /* Write data. */ + gcmkERR_BREAK( + gckOS_WriteMemory(Event->os, + gcmUINT64_TO_PTR(record->info.u.WriteData.address), + record->info.u.WriteData.data)); +#endif + break; + + case gcvHAL_UNLOCK_VIDEO_MEMORY: + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "gcvHAL_UNLOCK_VIDEO_MEMORY: 0x%x", + record->info.u.UnlockVideoMemory.node); + + nodeObject = gcmUINT64_TO_PTR(record->info.u.UnlockVideoMemory.node); + + /* Unlock, sync'ed. */ + status = gckVIDMEM_NODE_Unlock( + Event->kernel, + nodeObject, + record->processID, + gcvNULL + ); + + /* Deref node. */ + status = gckVIDMEM_NODE_Dereference(Event->kernel, nodeObject); + break; + + case gcvHAL_SIGNAL: + signal = gcmUINT64_TO_PTR(record->info.u.Signal.signal); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "gcvHAL_SIGNAL: 0x%x", + signal); + +#ifdef __QNXNTO__ + if ((record->info.u.Signal.coid == 0) + && (record->info.u.Signal.rcvid == 0) + ) + { + /* Kernel signal. */ + gcmkERR_BREAK( + gckOS_SignalPulse(Event->os, + signal)); + } + else + { + /* User signal. */ + gcmkERR_BREAK( + gckOS_UserSignal(Event->os, + signal, + record->info.u.Signal.rcvid, + record->info.u.Signal.coid)); + } +#else + /* Set signal. */ + if (gcmUINT64_TO_PTR(record->info.u.Signal.process) == gcvNULL) + { + /* Kernel signal. */ + gcmkERR_BREAK( + gckOS_Signal(Event->os, + signal, + gcvTRUE)); + } + else + { + /* User signal. */ + gcmkERR_BREAK( + gckOS_UserSignal(Event->os, + signal, + gcmUINT64_TO_PTR(record->info.u.Signal.process))); + } + + gcmkASSERT(record->info.u.Signal.auxSignal == 0); +#endif + break; + + case gcvHAL_TIMESTAMP: + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "gcvHAL_TIMESTAMP: %d %d", + record->info.u.TimeStamp.timer, + record->info.u.TimeStamp.request); + + /* Process the timestamp. */ + switch (record->info.u.TimeStamp.request) + { + case 0: + status = gckOS_GetTime(&Event->kernel->timers[ + record->info.u.TimeStamp.timer]. + stopTime); + break; + + case 1: + status = gckOS_GetTime(&Event->kernel->timers[ + record->info.u.TimeStamp.timer]. + startTime); + break; + + default: + gcmkTRACE_ZONE_N( + gcvLEVEL_ERROR, gcvZONE_EVENT, + gcmSIZEOF(record->info.u.TimeStamp.request), + "Invalid timestamp request: %d", + record->info.u.TimeStamp.request + ); + + status = gcvSTATUS_INVALID_ARGUMENT; + break; + } + break; + + case gcvHAL_COMMIT_DONE: + break; + + default: + /* Invalid argument. */ + gcmkTRACE_ZONE_N( + gcvLEVEL_ERROR, gcvZONE_EVENT, + gcmSIZEOF(record->info.command), + "Unknown event type: %d", + record->info.command + ); + + status = gcvSTATUS_INVALID_ARGUMENT; + break; + } + + /* Make sure there are no errors generated. */ + if (gcmIS_ERROR(status)) + { + gcmkTRACE_ZONE_N( + gcvLEVEL_WARNING, gcvZONE_EVENT, + gcmSIZEOF(status), + "Event produced status: %d(%s)", + status, gckOS_DebugStatus2Name(status)); + } + + /* Free the event. */ + gcmkVERIFY_OK(gckEVENT_FreeRecord(Event, record)); + + /* Advance to next record. */ + record = recordNext; + } + + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "Handled interrupt 0x%x", mask); + } + + if (IDs == 0) + { + gcmkONERROR(_TryToIdleGPU(Event)); + } + + /* End of event handling. */ + Event->notifyState = -1; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + } + + /* End of event handling. */ + Event->notifyState = -1; + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckEVENT_FreeProcess +** +** Free all events owned by a particular process ID. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctUINT32 ProcessID +** Process ID of the process to be freed up. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_FreeProcess( + IN gckEVENT Event, + IN gctUINT32 ProcessID + ) +{ + gctSIZE_T i; + gctBOOL acquired = gcvFALSE; + gcsEVENT_PTR record, next; + gceSTATUS status; + gcsEVENT_PTR deleteHead, deleteTail; + + gcmkHEADER_ARG("Event=0x%x ProcessID=%d", Event, ProcessID); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + /* Walk through all queues. */ + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if (Event->queues[i].head != gcvNULL) + { + /* Grab the event queue mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->eventQueueMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + /* Grab the mutex head. */ + record = Event->queues[i].head; + Event->queues[i].head = gcvNULL; + Event->queues[i].tail = gcvNULL; + deleteHead = gcvNULL; + deleteTail = gcvNULL; + + while (record != gcvNULL) + { + next = record->next; + if (record->processID == ProcessID) + { + if (deleteHead == gcvNULL) + { + deleteHead = record; + } + else + { + deleteTail->next = record; + } + + deleteTail = record; + } + else + { + if (Event->queues[i].head == gcvNULL) + { + Event->queues[i].head = record; + } + else + { + Event->queues[i].tail->next = record; + } + + Event->queues[i].tail = record; + } + + record->next = gcvNULL; + record = next; + } + + /* Release the mutex queue. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + + /* Loop through the entire list of events. */ + for (record = deleteHead; record != gcvNULL; record = next) + { + /* Get the next event record. */ + next = record->next; + + /* Free the event record. */ + gcmkONERROR(gckEVENT_FreeRecord(Event, record)); + } + } + } + + gcmkONERROR(_TryToIdleGPU(Event)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Release the event queue mutex. */ + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static void +_PrintRecord( + gcsEVENT_PTR record + ) +{ + switch (record->info.command) + { + case gcvHAL_WRITE_DATA: + gcmkPRINT(" gcvHAL_WRITE_DATA"); + break; + + case gcvHAL_UNLOCK_VIDEO_MEMORY: + gcmkPRINT(" gcvHAL_UNLOCK_VIDEO_MEMORY"); + break; + + case gcvHAL_SIGNAL: + gcmkPRINT(" gcvHAL_SIGNAL process=%lld signal=0x%llx", + record->info.u.Signal.process, + record->info.u.Signal.signal); + break; + + case gcvHAL_TIMESTAMP: + gcmkPRINT(" gcvHAL_TIMESTAMP"); + break; + + case gcvHAL_COMMIT_DONE: + gcmkPRINT(" gcvHAL_COMMIT_DONE"); + break; + + case gcvHAL_DESTROY_MMU: + gcmkPRINT(" gcvHAL_DESTORY_MMU mmu=%p", + gcmUINT64_TO_PTR(record->info.u.DestroyMmu.mmu)); + + break; + default: + gcmkPRINT(" Illegal Event %d", record->info.command); + break; + } +} + +/******************************************************************************* +** gckEVENT_Dump +** +** Dump record in event queue when stuck happens. +** No protection for the event queue. +**/ +gceSTATUS +gckEVENT_Dump( + IN gckEVENT Event + ) +{ + gcsEVENT_QUEUE_PTR queueHead = Event->queueHead; + gcsEVENT_QUEUE_PTR queue; + gcsEVENT_PTR record = gcvNULL; + gctINT i; +#if gcdINTERRUPT_STATISTIC + gctINT32 pendingInterrupt; + gctUINT32 intrAcknowledge; +#endif + gctINT32 pending; + + gcmkHEADER_ARG("Event=0x%x", Event); + + gcmkPRINT("**************************\n"); + gcmkPRINT("*** EVENT STATE DUMP ***\n"); + gcmkPRINT("**************************\n"); + + gcmkPRINT(" Unsumbitted Event:"); + while(queueHead) + { + queue = queueHead; + record = queueHead->head; + + gcmkPRINT(" [%p]:", queue); + while(record) + { + _PrintRecord(record); + record = record->next; + } + + if (queueHead == Event->queueTail) + { + queueHead = gcvNULL; + } + else + { + queueHead = queueHead->next; + } + } + + gcmkPRINT(" Untriggered Event:"); + for (i = 0; i < gcmCOUNTOF(Event->queues); i++) + { + queue = &Event->queues[i]; + record = queue->head; + + gcmkPRINT(" [%d]:", i); + while(record) + { + _PrintRecord(record); + record = record->next; + } + } + +#if gcdINTERRUPT_STATISTIC + gckOS_AtomGet(Event->os, Event->interruptCount, &pendingInterrupt); + gcmkPRINT(" Number of Pending Interrupt: %d", pendingInterrupt); + + if (Event->kernel->recovery == 0) + { + gceSTATUS status; + + status = gckOS_ReadRegisterEx( + Event->os, + Event->kernel->core, + 0x10, + &intrAcknowledge + ); + if (gcmIS_ERROR(status)) + { + gcmkPRINT(" READ INTR_ACKNOWLEDGE ERROR!"); + } + else + { + gcmkPRINT(" INTR_ACKNOWLEDGE=0x%x", intrAcknowledge); + } + } +#endif + + gcmkPRINT(" Notify State=%d", Event->notifyState); + + gckOS_AtomGet(Event->os, Event->pending, &pending); + + gcmkPRINT(" Pending=0x%x", pending); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +}
diff --git a/hal/kernel/gc_hal_kernel_heap.c b/hal/kernel/gc_hal_kernel_heap.c new file mode 100644 index 0000000..1e50050 --- /dev/null +++ b/hal/kernel/gc_hal_kernel_heap.c
@@ -0,0 +1,888 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +/** +** @file +** gckHEAP object for kernel HAL layer. The heap implemented here is an arena- +** based memory allocation. An arena-based memory heap allocates data quickly +** from specified arenas and reduces memory fragmentation. +** +*/ +#include "gc_hal_kernel_precomp.h" + +#define _GC_OBJ_ZONE gcvZONE_HEAP + +/******************************************************************************* +***** Structures *************************************************************** +*******************************************************************************/ +#define gcdIN_USE ((gcskNODE_PTR)gcvMAXUINTPTR_T) + +typedef struct _gcskNODE * gcskNODE_PTR; +typedef struct _gcskNODE +{ + /* Number of byets in node. */ + gctSIZE_T bytes; + + /* Pointer to next free node, or gcvNULL to mark the node as freed, or + ** gcdIN_USE to mark the node as used. */ + gcskNODE_PTR next; + +#if gcmIS_DEBUG(gcdDEBUG_CODE) + /* Time stamp of allocation. */ + gctUINT64 timeStamp; +#endif +} +gcskNODE; + +typedef struct _gcskHEAP * gcskHEAP_PTR; +typedef struct _gcskHEAP +{ + /* Linked list. */ + gcskHEAP_PTR next; + gcskHEAP_PTR prev; + + /* Heap size. */ + gctSIZE_T size; + + /* Free list. */ + gcskNODE_PTR freeList; +} +gcskHEAP; + +struct _gckHEAP +{ + /* Object. */ + gcsOBJECT object; + + /* Pointer to a gckOS object. */ + gckOS os; + + /* Locking mutex. */ + gctPOINTER mutex; + + /* Allocation parameters. */ + gctSIZE_T allocationSize; + + /* Heap list. */ + gcskHEAP_PTR heap; +#if gcmIS_DEBUG(gcdDEBUG_CODE) + gctUINT64 timeStamp; +#endif + +#if VIVANTE_PROFILER_SYSTEM_MEMORY || gcmIS_DEBUG(gcdDEBUG_CODE) + /* Profile information. */ + gctUINT32 allocCount; + gctUINT64 allocBytes; + gctUINT64 allocBytesMax; + gctUINT64 allocBytesTotal; + gctUINT32 heapCount; + gctUINT32 heapCountMax; + gctUINT64 heapMemory; + gctUINT64 heapMemoryMax; +#endif +}; + +/******************************************************************************* +***** Static Support Functions ************************************************* +*******************************************************************************/ + +#if gcmIS_DEBUG(gcdDEBUG_CODE) +static gctSIZE_T +_DumpHeap( + IN gcskHEAP_PTR Heap + ) +{ + gctPOINTER p; + gctSIZE_T leaked = 0; + + /* Start at first node. */ + for (p = Heap + 1;;) + { + /* Convert the pointer. */ + gcskNODE_PTR node = (gcskNODE_PTR) p; + + /* Check if this is a used node. */ + if (node->next == gcdIN_USE) + { + /* Print the leaking node. */ + gcmkTRACE_ZONE(gcvLEVEL_WARNING, gcvZONE_HEAP, + "Detected leaking: node=0x%x bytes=%lu timeStamp=%llu " + "(%08X %c%c%c%c)", + node, node->bytes, node->timeStamp, + ((gctUINT32_PTR) (node + 1))[0], + gcmPRINTABLE(((gctUINT8_PTR) (node + 1))[0]), + gcmPRINTABLE(((gctUINT8_PTR) (node + 1))[1]), + gcmPRINTABLE(((gctUINT8_PTR) (node + 1))[2]), + gcmPRINTABLE(((gctUINT8_PTR) (node + 1))[3])); + + /* Add leaking byte count. */ + leaked += node->bytes; + } + + /* Test for end of heap. */ + if (node->bytes == 0) + { + break; + } + + else + { + /* Move to next node. */ + p = (gctUINT8_PTR) node + node->bytes; + } + } + + /* Return the number of leaked bytes. */ + return leaked; +} +#endif + +static gceSTATUS +_CompactKernelHeap( + IN gckHEAP Heap + ) +{ + gcskHEAP_PTR heap, next; + gctPOINTER p; + gcskHEAP_PTR freeList = gcvNULL; + + gcmkHEADER_ARG("Heap=0x%x", Heap); + + /* Walk all the heaps. */ + for (heap = Heap->heap; heap != gcvNULL; heap = next) + { + gcskNODE_PTR lastFree = gcvNULL; + + /* Zero out the free list. */ + heap->freeList = gcvNULL; + + /* Start at the first node. */ + for (p = (gctUINT8_PTR) (heap + 1);;) + { + /* Convert the pointer. */ + gcskNODE_PTR node = (gcskNODE_PTR) p; + + gcmkASSERT(p <= (gctPOINTER) ((gctUINT8_PTR) (heap + 1) + heap->size)); + + /* Test if this node not used. */ + if (node->next != gcdIN_USE) + { + /* Test if this is the end of the heap. */ + if (node->bytes == 0) + { + break; + } + + /* Test of this is the first free node. */ + else if (lastFree == gcvNULL) + { + /* Initialzie the free list. */ + heap->freeList = node; + lastFree = node; + } + + else + { + /* Test if this free node is contiguous with the previous + ** free node. */ + if ((gctUINT8_PTR) lastFree + lastFree->bytes == p) + { + /* Just increase the size of the previous free node. */ + lastFree->bytes += node->bytes; + } + else + { + /* Add to linked list. */ + lastFree->next = node; + lastFree = node; + } + } + } + + /* Move to next node. */ + p = (gctUINT8_PTR) node + node->bytes; + } + + /* Mark the end of the chain. */ + if (lastFree != gcvNULL) + { + lastFree->next = gcvNULL; + } + + /* Get next heap. */ + next = heap->next; + + /* Check if the entire heap is free. */ + if ((heap->freeList != gcvNULL) + && (heap->freeList->bytes == heap->size - gcmSIZEOF(gcskNODE)) + ) + { + /* Remove the heap from the linked list. */ + if (heap->prev == gcvNULL) + { + Heap->heap = next; + } + else + { + heap->prev->next = next; + } + + if (heap->next != gcvNULL) + { + heap->next->prev = heap->prev; + } + +#if VIVANTE_PROFILER_SYSTEM_MEMORY || gcmIS_DEBUG(gcdDEBUG_CODE) + /* Update profiling. */ + Heap->heapCount -= 1; + Heap->heapMemory -= heap->size + gcmSIZEOF(gcskHEAP); +#endif + + /* Add this heap to the list of heaps that need to be freed. */ + heap->next = freeList; + freeList = heap; + } + } + + if (freeList != gcvNULL) + { + /* Release the mutex, remove any chance for a dead lock. */ + gcmkVERIFY_OK( + gckOS_ReleaseMutex(Heap->os, Heap->mutex)); + + /* Free all heaps in the free list. */ + for (heap = freeList; heap != gcvNULL; heap = next) + { + /* Get pointer to the next heap. */ + next = heap->next; + + /* Free the heap. */ + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HEAP, + "Freeing heap 0x%x (%lu bytes)", + heap, heap->size + gcmSIZEOF(gcskHEAP)); + gcmkVERIFY_OK(gckOS_FreeMemory(Heap->os, heap)); + } + + /* Acquire the mutex again. */ + gcmkVERIFY_OK( + gckOS_AcquireMutex(Heap->os, Heap->mutex, gcvINFINITE)); + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +***** gckHEAP API Code ********************************************************* +*******************************************************************************/ + +/******************************************************************************* +** +** gckHEAP_Construct +** +** Construct a new gckHEAP object. +** +** INPUT: +** +** gckOS Os +** Pointer to a gckOS object. +** +** gctSIZE_T AllocationSize +** Minimum size per arena. +** +** OUTPUT: +** +** gckHEAP * Heap +** Pointer to a variable that will hold the pointer to the gckHEAP +** object. +*/ +gceSTATUS +gckHEAP_Construct( + IN gckOS Os, + IN gctSIZE_T AllocationSize, + OUT gckHEAP * Heap + ) +{ + gceSTATUS status; + gckHEAP heap = gcvNULL; + gctPOINTER pointer = gcvNULL; + + gcmkHEADER_ARG("Os=0x%x AllocationSize=%lu", Os, AllocationSize); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Os, gcvOBJ_OS); + gcmkVERIFY_ARGUMENT(Heap != gcvNULL); + + /* Allocate the gckHEAP object. */ + gcmkONERROR(gckOS_AllocateMemory(Os, + gcmSIZEOF(struct _gckHEAP), + &pointer)); + + heap = pointer; + + /* Initialize the gckHEAP object. */ + heap->object.type = gcvOBJ_HEAP; + heap->os = Os; + heap->allocationSize = AllocationSize; + heap->heap = gcvNULL; +#if gcmIS_DEBUG(gcdDEBUG_CODE) + heap->timeStamp = 0; +#endif + +#if VIVANTE_PROFILER_SYSTEM_MEMORY || gcmIS_DEBUG(gcdDEBUG_CODE) + /* Zero the counters. */ + heap->allocCount = 0; + heap->allocBytes = 0; + heap->allocBytesMax = 0; + heap->allocBytesTotal = 0; + heap->heapCount = 0; + heap->heapCountMax = 0; + heap->heapMemory = 0; + heap->heapMemoryMax = 0; +#endif + + /* Create the mutex. */ + gcmkONERROR(gckOS_CreateMutex(Os, &heap->mutex)); + + /* Return the pointer to the gckHEAP object. */ + *Heap = heap; + + /* Success. */ + gcmkFOOTER_ARG("*Heap=0x%x", *Heap); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (heap != gcvNULL) + { + /* Free the heap structure. */ + gcmkVERIFY_OK(gckOS_FreeMemory(Os, heap)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckHEAP_Destroy +** +** Destroy a gckHEAP object. +** +** INPUT: +** +** gckHEAP Heap +** Pointer to a gckHEAP object to destroy. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckHEAP_Destroy( + IN gckHEAP Heap + ) +{ + gcskHEAP_PTR heap; +#if gcmIS_DEBUG(gcdDEBUG_CODE) + gctSIZE_T leaked = 0; +#endif + + gcmkHEADER_ARG("Heap=0x%x", Heap); + + for (heap = Heap->heap; heap != gcvNULL; heap = Heap->heap) + { + /* Unlink heap from linked list. */ + Heap->heap = heap->next; + +#if gcmIS_DEBUG(gcdDEBUG_CODE) + /* Check for leaked memory. */ + leaked += _DumpHeap(heap); +#endif + + /* Free the heap. */ + gcmkVERIFY_OK(gckOS_FreeMemory(Heap->os, heap)); + } + + /* Free the mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Heap->os, Heap->mutex)); + + /* Free the heap structure. */ + gcmkVERIFY_OK(gckOS_FreeMemory(Heap->os, Heap)); + + /* Success. */ +#if gcmIS_DEBUG(gcdDEBUG_CODE) + gcmkFOOTER_ARG("leaked=%lu", leaked); +#else + gcmkFOOTER_NO(); +#endif + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckHEAP_Allocate +** +** Allocate data from the heap. +** +** INPUT: +** +** gckHEAP Heap +** Pointer to a gckHEAP object. +** +** IN gctSIZE_T Bytes +** Number of byte to allocate. +** +** OUTPUT: +** +** gctPOINTER * Memory +** Pointer to a variable that will hold the address of the allocated +** memory. +*/ +gceSTATUS +gckHEAP_Allocate( + IN gckHEAP Heap, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Memory + ) +{ + gctBOOL acquired = gcvFALSE; + gcskHEAP_PTR heap; + gceSTATUS status; + gctSIZE_T bytes; + gcskNODE_PTR node, used, prevFree = gcvNULL; + gctPOINTER memory = gcvNULL; + + gcmkHEADER_ARG("Heap=0x%x Bytes=%lu", Heap, Bytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Heap, gcvOBJ_HEAP); + gcmkVERIFY_ARGUMENT(Bytes > 0); + gcmkVERIFY_ARGUMENT(Memory != gcvNULL); + + /* Determine number of bytes required for a node. */ + bytes = gcmALIGN(Bytes + gcmSIZEOF(gcskNODE), 8); + + /* Acquire the mutex. */ + gcmkONERROR( + gckOS_AcquireMutex(Heap->os, Heap->mutex, gcvINFINITE)); + + acquired = gcvTRUE; + + /* Check if this allocation is bigger than the default allocation size. */ + if (bytes > Heap->allocationSize - gcmSIZEOF(gcskHEAP) - gcmSIZEOF(gcskNODE)) + { + /* Adjust allocation size. */ + Heap->allocationSize = bytes * 2; + } + + else if (Heap->heap != gcvNULL) + { + gctINT i; + + /* 2 retries, since we might need to compact. */ + for (i = 0; i < 2; ++i) + { + /* Walk all the heaps. */ + for (heap = Heap->heap; heap != gcvNULL; heap = heap->next) + { + /* Check if this heap has enough bytes to hold the request. */ + if (bytes <= heap->size - gcmSIZEOF(gcskNODE)) + { + prevFree = gcvNULL; + + /* Walk the chain of free nodes. */ + for (node = heap->freeList; + node != gcvNULL; + node = node->next + ) + { + gcmkASSERT(node->next != gcdIN_USE); + + /* Check if this free node has enough bytes. */ + if (node->bytes >= bytes) + { + /* Use the node. */ + goto UseNode; + } + + /* Save current free node for linked list management. */ + prevFree = node; + } + } + } + + if (i == 0) + { + /* Compact the heap. */ + gcmkVERIFY_OK(_CompactKernelHeap(Heap)); + +#if gcmIS_DEBUG(gcdDEBUG_CODE) + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_HEAP, + "===== KERNEL HEAP ====="); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_HEAP, + "Number of allocations : %12u", + Heap->allocCount); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_HEAP, + "Number of bytes allocated : %12llu", + Heap->allocBytes); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_HEAP, + "Maximum allocation size : %12llu", + Heap->allocBytesMax); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_HEAP, + "Total number of bytes allocated : %12llu", + Heap->allocBytesTotal); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_HEAP, + "Number of heaps : %12u", + Heap->heapCount); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_HEAP, + "Heap memory in bytes : %12llu", + Heap->heapMemory); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_HEAP, + "Maximum number of heaps : %12u", + Heap->heapCountMax); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_HEAP, + "Maximum heap memory in bytes : %12llu", + Heap->heapMemoryMax); +#endif + } + } + } + + /* Release the mutex. */ + gcmkONERROR( + gckOS_ReleaseMutex(Heap->os, Heap->mutex)); + + acquired = gcvFALSE; + + /* Allocate a new heap. */ + gcmkONERROR( + gckOS_AllocateMemory(Heap->os, + Heap->allocationSize, + &memory)); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_HEAP, + "Allocated heap 0x%x (%lu bytes)", + memory, Heap->allocationSize); + + /* Acquire the mutex. */ + gcmkONERROR( + gckOS_AcquireMutex(Heap->os, Heap->mutex, gcvINFINITE)); + + acquired = gcvTRUE; + + /* Use the allocated memory as the heap. */ + heap = (gcskHEAP_PTR) memory; + + /* Insert this heap to the head of the chain. */ + heap->next = Heap->heap; + heap->prev = gcvNULL; + heap->size = Heap->allocationSize - gcmSIZEOF(gcskHEAP); + + if (heap->next != gcvNULL) + { + heap->next->prev = heap; + } + Heap->heap = heap; + + /* Mark the end of the heap. */ + node = (gcskNODE_PTR) ( (gctUINT8_PTR) heap + + Heap->allocationSize + - gcmSIZEOF(gcskNODE) + ); + node->bytes = 0; + node->next = gcvNULL; + + /* Create a free list. */ + node = (gcskNODE_PTR) (heap + 1); + heap->freeList = node; + + /* Initialize the free list. */ + node->bytes = heap->size - gcmSIZEOF(gcskNODE); + node->next = gcvNULL; + + /* No previous free. */ + prevFree = gcvNULL; + +#if VIVANTE_PROFILER_SYSTEM_MEMORY || gcmIS_DEBUG(gcdDEBUG_CODE) + /* Update profiling. */ + Heap->heapCount += 1; + Heap->heapMemory += Heap->allocationSize; + + if (Heap->heapCount > Heap->heapCountMax) + { + Heap->heapCountMax = Heap->heapCount; + } + if (Heap->heapMemory > Heap->heapMemoryMax) + { + Heap->heapMemoryMax = Heap->heapMemory; + } +#endif + +UseNode: + /* Verify some stuff. */ + gcmkASSERT(heap != gcvNULL); + gcmkASSERT(node != gcvNULL); + gcmkASSERT(node->bytes >= bytes); + + if (heap->prev != gcvNULL) + { + /* Unlink the heap from the linked list. */ + heap->prev->next = heap->next; + if (heap->next != gcvNULL) + { + heap->next->prev = heap->prev; + } + + /* Move the heap to the front of the list. */ + heap->next = Heap->heap; + heap->prev = gcvNULL; + Heap->heap = heap; + heap->next->prev = heap; + } + + /* Check if there is enough free space left after usage for another free + ** node. */ + if (node->bytes - bytes >= gcmSIZEOF(gcskNODE)) + { + /* Allocated used space from the back of the free list. */ + used = (gcskNODE_PTR) ((gctUINT8_PTR) node + node->bytes - bytes); + + /* Adjust the number of free bytes. */ + node->bytes -= bytes; + gcmkASSERT(node->bytes >= gcmSIZEOF(gcskNODE)); + } + else + { + /* Remove this free list from the chain. */ + if (prevFree == gcvNULL) + { + heap->freeList = node->next; + } + else + { + prevFree->next = node->next; + } + + /* Consume the entire free node. */ + used = (gcskNODE_PTR) node; + bytes = node->bytes; + } + + /* Mark node as used. */ + used->bytes = bytes; + used->next = gcdIN_USE; +#if gcmIS_DEBUG(gcdDEBUG_CODE) + used->timeStamp = ++Heap->timeStamp; +#endif + +#if VIVANTE_PROFILER_SYSTEM_MEMORY || gcmIS_DEBUG(gcdDEBUG_CODE) + /* Update profile counters. */ + Heap->allocCount += 1; + Heap->allocBytes += bytes; + Heap->allocBytesMax = gcmMAX(Heap->allocBytes, Heap->allocBytesMax); + Heap->allocBytesTotal += bytes; +#endif + + /* Release the mutex. */ + gcmkVERIFY_OK( + gckOS_ReleaseMutex(Heap->os, Heap->mutex)); + + /* Return pointer to memory. */ + *Memory = used + 1; + + /* Success. */ + gcmkFOOTER_ARG("*Memory=0x%x", *Memory); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK( + gckOS_ReleaseMutex(Heap->os, Heap->mutex)); + } + + if (memory != gcvNULL) + { + /* Free the heap memory. */ + gckOS_FreeMemory(Heap->os, memory); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckHEAP_Free +** +** Free allocated memory from the heap. +** +** INPUT: +** +** gckHEAP Heap +** Pointer to a gckHEAP object. +** +** IN gctPOINTER Memory +** Pointer to memory to free. +** +** OUTPUT: +** +** NOTHING. +*/ +gceSTATUS +gckHEAP_Free( + IN gckHEAP Heap, + IN gctPOINTER Memory + ) +{ + gcskNODE_PTR node; + gceSTATUS status; + + gcmkHEADER_ARG("Heap=0x%x Memory=0x%x", Heap, Memory); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Heap, gcvOBJ_HEAP); + gcmkVERIFY_ARGUMENT(Memory != gcvNULL); + + /* Acquire the mutex. */ + gcmkONERROR( + gckOS_AcquireMutex(Heap->os, Heap->mutex, gcvINFINITE)); + + /* Pointer to structure. */ + node = (gcskNODE_PTR) Memory - 1; + + /* Mark the node as freed. */ + node->next = gcvNULL; + +#if VIVANTE_PROFILER_SYSTEM_MEMORY || gcmIS_DEBUG(gcdDEBUG_CODE) + /* Update profile counters. */ + Heap->allocBytes -= node->bytes; +#endif + + /* Release the mutex. */ + gcmkVERIFY_OK( + gckOS_ReleaseMutex(Heap->os, Heap->mutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +#if VIVANTE_PROFILER_SYSTEM_MEMORY +gceSTATUS +gckHEAP_ProfileStart( + IN gckHEAP Heap + ) +{ + gcmkHEADER_ARG("Heap=0x%x", Heap); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Heap, gcvOBJ_HEAP); + + /* Zero the counters. */ + Heap->allocCount = 0; + Heap->allocBytes = 0; + Heap->allocBytesMax = 0; + Heap->allocBytesTotal = 0; + Heap->heapCount = 0; + Heap->heapCountMax = 0; + Heap->heapMemory = 0; + Heap->heapMemoryMax = 0; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckHEAP_ProfileEnd( + IN gckHEAP Heap, + IN gctCONST_STRING Title + ) +{ + gcmkHEADER_ARG("Heap=0x%x Title=0x%x", Heap, Title); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Heap, gcvOBJ_HEAP); + gcmkVERIFY_ARGUMENT(Title != gcvNULL); + + gcmkPRINT("\n"); + gcmkPRINT("=====[ HEAP - %s ]=====", Title); + gcmkPRINT("Number of allocations : %12u", Heap->allocCount); + gcmkPRINT("Number of bytes allocated : %12llu", Heap->allocBytes); + gcmkPRINT("Maximum allocation size : %12llu", Heap->allocBytesMax); + gcmkPRINT("Total number of bytes allocated : %12llu", Heap->allocBytesTotal); + gcmkPRINT("Number of heaps : %12u", Heap->heapCount); + gcmkPRINT("Heap memory in bytes : %12llu", Heap->heapMemory); + gcmkPRINT("Maximum number of heaps : %12u", Heap->heapCountMax); + gcmkPRINT("Maximum heap memory in bytes : %12llu", Heap->heapMemoryMax); + gcmkPRINT("=============================================="); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} +#endif /* VIVANTE_PROFILER_SYSTEM_MEMORY */ +
diff --git a/hal/kernel/gc_hal_kernel_mmu.c b/hal/kernel/gc_hal_kernel_mmu.c new file mode 100644 index 0000000..714ff28 --- /dev/null +++ b/hal/kernel/gc_hal_kernel_mmu.c
@@ -0,0 +1,3422 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" + +#define _GC_OBJ_ZONE gcvZONE_MMU + +typedef enum _gceMMU_TYPE +{ + gcvMMU_USED = (0 << 4), + gcvMMU_SINGLE = (1 << 4), + gcvMMU_FREE = (2 << 4), +} +gceMMU_TYPE; + +/* VIP SRAM start virtual address. */ +#define gcdRESERVE_START (4 << 20) +#define gcdRESERVE_ALIGN (4 << 10) + +#define gcmENTRY_TYPE(x) (x & 0xF0) + +#define gcmENTRY_COUNT(x) ((x & 0xFFFFFF00) >> 8) + +#define gcdMMU_TABLE_DUMP 0 + +#define gcdVERTEX_START (128 << 10) + +typedef struct _gcsMMU_STLB_CHUNK *gcsMMU_STLB_CHUNK_PTR; + +typedef struct _gcsMMU_STLB_CHUNK +{ + gckVIDMEM_NODE videoMem; + gctUINT32_PTR logical; + gctSIZE_T size; + gctPHYS_ADDR_T physBase; + gctSIZE_T pageCount; + gctUINT32 mtlbIndex; + gctUINT32 mtlbEntryNum; + gcsMMU_STLB_CHUNK_PTR next; +} gcsMMU_STLB_CHUNK; + +#if gcdSHARED_PAGETABLE +typedef struct _gcsSharedPageTable * gcsSharedPageTable_PTR; +typedef struct _gcsSharedPageTable +{ + /* Shared gckMMU object. */ + gckMMU mmu; + + /* Hardwares which use this shared pagetable. */ + gckHARDWARE hardwares[gcdMAX_GPU_COUNT]; + + /* Number of cores use this shared pagetable. */ + gctUINT32 reference; +} +gcsSharedPageTable; + +static gcsSharedPageTable_PTR sharedPageTable = gcvNULL; +#endif + +typedef struct _gcsFreeSpaceNode * gcsFreeSpaceNode_PTR; +typedef struct _gcsFreeSpaceNode +{ + gctUINT32 start; + gctINT32 entries; +} +gcsFreeSpaceNode; + +#if gcdENDIAN_BIG + +# define _WritePageEntry(pageEntry, entryValue) \ + *(gctUINT32_PTR)(pageEntry) = gcmBSWAP32((gctUINT32)(entryValue)) + +# define _ReadPageEntry(pageEntry) \ + gcmBSWAP32(*(gctUINT32_PTR)(pageEntry)) + +#else + +# define _WritePageEntry(pageEntry, entryValue) \ + *(gctUINT32_PTR)(pageEntry) = (gctUINT32)(entryValue) + +# define _ReadPageEntry(pageEntry) \ + *(gctUINT32_PTR)(pageEntry) + +#endif + +static gceSTATUS +_FillPageTable( + IN gctUINT32_PTR PageTable, + IN gctUINT32 PageCount, + IN gctUINT32 EntryValue +) +{ + gctUINT i; + + for (i = 0; i < PageCount; i++) + { + _WritePageEntry(PageTable + i, EntryValue); + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_FillMap( + IN gctUINT32_PTR Map, + IN gctUINT32 PageCount, + IN gctUINT32 EntryValue +) +{ + gctUINT i; + + for (i = 0; i < PageCount; i++) + { + Map[i] = EntryValue; + } + + return gcvSTATUS_OK; +} + +static gceSTATUS +_Link( + IN gcsADDRESS_AREA_PTR Area, + IN gctUINT32 Index, + IN gctUINT32 Next + ) +{ + if (Index >= Area->stlbEntries) + { + /* Just move heap pointer. */ + Area->heapList = Next; + } + else + { + /* Address page table. */ + gctUINT32_PTR map = Area->mapLogical; + + /* Dispatch on node type. */ + switch (gcmENTRY_TYPE(map[Index])) + { + case gcvMMU_SINGLE: + /* Set single index. */ + map[Index] = (Next << 8) | gcvMMU_SINGLE; + break; + + case gcvMMU_FREE: + /* Set index. */ + map[Index + 1] = Next; + break; + + default: + gcmkFATAL("MMU table correcupted at index %u!", Index); + return gcvSTATUS_HEAP_CORRUPTED; + } + } + + /* Success. */ + return gcvSTATUS_OK; +} + +static gceSTATUS +_AddFree( + IN gcsADDRESS_AREA_PTR Area, + IN gctUINT32 Index, + IN gctUINT32 Node, + IN gctUINT32 Count + ) +{ + gctUINT32_PTR map = Area->mapLogical; + + if (Count == 1) + { + /* Initialize a single page node. */ + map[Node] = (~((1U<<8)-1)) | gcvMMU_SINGLE; + } + else + { + /* Initialize the node. */ + map[Node + 0] = (Count << 8) | gcvMMU_FREE; + map[Node + 1] = ~0U; + } + + /* Append the node. */ + return _Link(Area, Index, Node); +} + +static gceSTATUS +_Collect( + IN gcsADDRESS_AREA_PTR Area + ) +{ + gctUINT32_PTR map = Area->mapLogical; + gceSTATUS status; + gctUINT32 i, previous, start = 0, count = 0; + + previous = Area->heapList = ~0U; + Area->freeNodes = gcvFALSE; + + /* Walk the entire page table. */ + for (i = 0; i < Area->stlbEntries; ++i) + { + /* Dispatch based on type of page. */ + switch (gcmENTRY_TYPE(map[i])) + { + case gcvMMU_USED: + /* Used page, so close any open node. */ + if (count > 0) + { + /* Add the node. */ + gcmkONERROR(_AddFree(Area, previous, start, count)); + + /* Reset the node. */ + previous = start; + count = 0; + } + break; + + case gcvMMU_SINGLE: + /* Single free node. */ + if (count++ == 0) + { + /* Start a new node. */ + start = i; + } + break; + + case gcvMMU_FREE: + /* A free node. */ + if (count == 0) + { + /* Start a new node. */ + start = i; + } + + /* Advance the count. */ + count += map[i] >> 8; + + /* Advance the index into the page table. */ + i += (map[i] >> 8) - 1; + break; + + default: + gcmkFATAL("MMU page table correcupted at index %u!", i); + return gcvSTATUS_HEAP_CORRUPTED; + } + } + + /* See if we have an open node left. */ + if (count > 0) + { + /* Add the node to the list. */ + gcmkONERROR(_AddFree(Area, previous, start, count)); + } + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_MMU, + "Performed a garbage collection of the MMU heap."); + + /* Success. */ + return gcvSTATUS_OK; + +OnError: + /* Return the staus. */ + return status; +} + +static gctUINT32 +_SetPage(gctUINT32 PageAddress, gctUINT32 PageAddressExt, gctBOOL Writable) +{ + gctUINT32 entry = PageAddress + /* AddressExt */ + | (PageAddressExt << 4) + /* Ignore exception */ + | (0 << 1) + /* Present */ + | (1 << 0); + + if (Writable) + { + /* writable */ + entry |= (1 << 2); + } +#if gcdUSE_MMU_EXCEPTION + else + { + /* If this page is read only, set exception bit to make exception happens + ** when writing to it. */ + entry |= gcdMMU_STLB_EXCEPTION; + } +#endif + + return entry; +} + +static gctUINT32 +_MtlbOffset( + gctUINT32 Address + ) +{ + return (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; +} + +gctUINT32 +_AddressToIndex( + IN gcsADDRESS_AREA_PTR Area, + IN gctUINT32 Address + ) +{ + + gctUINT32 stlbShift = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT; + gctUINT32 stlbMask = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_MASK : gcdMMU_STLB_4K_MASK; + gctUINT32 stlbEntryNum = (Area->areaType == gcvAREA_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM; + gctUINT32 mtlbOffset = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; + gctUINT32 stlbOffset = (Address & stlbMask) >> stlbShift; + + return (mtlbOffset - Area->mappingStart) * stlbEntryNum + stlbOffset; +} + +static gctUINT32_PTR +_StlbEntry( + gcsADDRESS_AREA_PTR Area, + gctUINT32 Address + ) +{ + gctUINT32 index = _AddressToIndex(Area, Address); + + return &Area->stlbLogical[index]; +} + +static gctBOOL +_IsRangeInsected( + gctUINT64 baseAddress1, + gctSIZE_T size1, + gctUINT64 baseAddress2, + gctSIZE_T size2 + ) +{ + gctUINT64 endAddress1 = baseAddress1 + size1 - 1; + gctUINT64 endAddress2 = baseAddress2 + size2 - 1; + + if (!size1 || !size2) + { + return gcvFALSE; + } + + return (((baseAddress2 <= endAddress1) && (endAddress2 >= baseAddress1)) || + ((baseAddress1 <= endAddress2) && (endAddress1 >= baseAddress2))); +} + +static gceSTATUS +_FillFlatMappingInMap( + gcsADDRESS_AREA_PTR Area, + gctUINT32 Index, + gctUINT32 NumPages + ) +{ + gceSTATUS status; + gctUINT32 i; + gctBOOL gotIt = gcvFALSE; + gctUINT32 index = Index; + gctUINT32_PTR map = Area->mapLogical; + gctUINT32 previous = ~0U; + + /* Find node which contains index. */ + for (i = 0; !gotIt && (i < Area->stlbEntries);) + { + gctUINT32 numPages; + + switch (gcmENTRY_TYPE(map[i])) + { + case gcvMMU_SINGLE: + if (i == index) + { + gotIt = gcvTRUE; + } + else + { + previous = i; + i = map[i] >> 8; + } + break; + + case gcvMMU_FREE: + numPages = map[i] >> 8; + if (index >= i && index + NumPages - 1 < i + numPages) + { + gotIt = gcvTRUE; + } + else + { + previous = i; + i = map[i + 1]; + } + break; + + case gcvMMU_USED: + i++; + break; + + default: + gcmkFATAL("MMU table correcupted at index %u!", index); + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + } + + switch (gcmENTRY_TYPE(map[i])) + { + case gcvMMU_SINGLE: + /* Unlink single node from free list. */ + gcmkONERROR( + _Link(Area, previous, map[i] >> 8)); + break; + + case gcvMMU_FREE: + /* Split the node. */ + { + gctUINT32 start; + gctUINT32 next = map[i+1]; + gctUINT32 total = map[i] >> 8; + gctUINT32 countLeft = index - i; + gctUINT32 countRight = total - countLeft - NumPages; + + if (countLeft) + { + start = i; + _AddFree(Area, previous, start, countLeft); + previous = start; + } + + if (countRight) + { + start = index + NumPages; + _AddFree(Area, previous, start, countRight); + previous = start; + } + + _Link(Area, previous, next); + } + break; + } + + _FillMap(&map[index], NumPages, gcvMMU_USED); + + return gcvSTATUS_OK; +OnError: + return status; +} + +static gceSTATUS +_CollectFreeSpace( + IN gckMMU Mmu, + OUT gcsFreeSpaceNode_PTR *Array, + OUT gctINT * Size + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctPOINTER pointer = gcvNULL; + gcsFreeSpaceNode_PTR array = gcvNULL; + gcsFreeSpaceNode_PTR node = gcvNULL; + gctINT size = 0; + gctINT i = 0; + + for (i = 0; i < gcdMMU_MTLB_ENTRY_NUM; i++) + { + if (!Mmu->mtlbLogical[i]) + { + if (!node) + { + /* This is the first entry of the free space. */ + node += 1; + size++; + + } + } + else if (node) + { + /* Reset the start. */ + node = gcvNULL; + } + } + + /* Allocate memory for the array. */ + gcmkONERROR(gckOS_Allocate(Mmu->os, + gcmSIZEOF(*array) * size, + &pointer)); + + array = (gcsFreeSpaceNode_PTR)pointer; + node = gcvNULL; + + for (i = 0, size = 0; i < gcdMMU_MTLB_ENTRY_NUM; i++) + { + if (!Mmu->mtlbLogical[i]) + { + if (!node) + { + /* This is the first entry of the free space. */ + node = &array[size++]; + + node->start = i; + node->entries = 0; + } + + node->entries++; + } + else if (node) + { + /* Reset the start. */ + node = gcvNULL; + } + } + +#if gcdMMU_TABLE_DUMP + for (i = 0; i < size; i++) + { + gckOS_Print("%s(%d): [%d]: start=%d, entries=%d.\n", + __FUNCTION__, __LINE__, + i, + array[i].start, + array[i].entries); + } +#endif + + *Array = array; + *Size = size; + + return gcvSTATUS_OK; + +OnError: + if (pointer != gcvNULL) + { + gckOS_Free(Mmu->os, pointer); + } + + return status; +} + +gceSTATUS +_GetMtlbFreeSpace( + IN gckMMU Mmu, + IN gctUINT32 NumEntries, + OUT gctUINT32 *MtlbStart, + OUT gctUINT32 *MtlbEnd + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gcsFreeSpaceNode_PTR nodeArray = gcvNULL; + gctINT i, nodeArraySize = 0; + gctINT numEntries = gcdMMU_MTLB_ENTRY_NUM; + gctINT32 mStart = -1; + gctINT32 mEnd = -1; + + gcmkONERROR(_CollectFreeSpace(Mmu, &nodeArray, &nodeArraySize)); + + /* Find the smallest space for NumEntries */ + for (i = 0; i < nodeArraySize; i++) + { + if (nodeArray[i].entries < numEntries && NumEntries <= (gctUINT32)nodeArray[i].entries) + { + numEntries = nodeArray[i].entries; + + mStart = nodeArray[i].start; + mEnd = nodeArray[i].start + NumEntries - 1; + } + } + + if (mStart == -1 && mEnd == -1) + { + gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY); + } + + *MtlbStart = (gctUINT32)mStart; + *MtlbEnd = (gctUINT32)mEnd; + +OnError: + if (nodeArray) + { + gckOS_Free(Mmu->os, (gctPOINTER)nodeArray); + } + + return status; +} + +static gcePOOL _GetPageTablePool(IN gckOS Os) +{ + gcePOOL pool = gcvPOOL_DEFAULT; + gctUINT64 data = 0; + gceSTATUS status; + + status = gckOS_QueryOption(Os, "mmuPageTablePool", &data); + + if (status == gcvSTATUS_OK) + { + if (data == 1) + { + pool = gcvPOOL_VIRTUAL; + } + } + + return pool; +} + +static gceSTATUS +_FillFlatMapping( + IN gckMMU Mmu, + IN gctUINT64 PhysBase, + IN gctSIZE_T Size, + IN gctBOOL reserved, + IN gctBOOL ableToShift, + OUT gctUINT32 *GpuBaseAddress + ) +{ + gceSTATUS status; + gctUINT32 mtlb; + gctUINT32 physBase; + gckKERNEL kernel = Mmu->hardware->kernel; + gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K; + gctBOOL physicalRangeOverlapped = gcvFALSE; + gctBOOL virtualRangeOverlapped = gcvFALSE; + gctBOOL specificFlatMapping = gcvFALSE; + gctBOOL needShiftMapping = gcvFALSE; + gctUINT64 flatBase = PhysBase; + gctUINT32 flatSize = (gctUINT32) Size; + gctUINT64 base = flatBase; + gctUINT64 end = base + flatSize; + gctUINT32 reqVirtualBase = 0; + gctUINT32 flatVirtualBase = 0; + gctUINT32 i; + + /************************ Get flat mapping type and range. ************************/ + { + for (i = 0; i < Mmu->gpuPhysicalRangeCount; i++) + { + if (base < Mmu->gpuPhysicalRanges[i].start) + { + if (end > Mmu->gpuPhysicalRanges[i].start) + { + physicalRangeOverlapped = gcvTRUE; + if (Mmu->gpuPhysicalRanges[i].flag == gcvFLATMAP_DIRECT) + { + /* Overlapped part is direct mapping, continue direct mapping */ + end = Mmu->gpuPhysicalRanges[i].start; + } + else + { + /* Overlapped part is shift mapping, do entire shift mapping */ + needShiftMapping = gcvTRUE; + } + } + + flatSize = (gctUINT32) (end - base); + } + else if (end > Mmu->gpuPhysicalRanges[i].end) + { + if (base < Mmu->gpuPhysicalRanges[i].end) + { + physicalRangeOverlapped = gcvTRUE; + if (Mmu->gpuPhysicalRanges[i].flag == gcvFLATMAP_DIRECT) + { + /* Overlapped part is direct mapping, continue direct mapping */ + base = Mmu->gpuPhysicalRanges[i].end + 1; + } + else + { + /* Overlapped part is shift mapping, do entire shift mapping */ + needShiftMapping = gcvTRUE; + } + + } + + flatBase = base; + flatSize = (gctUINT32) (end - base); + } + else + { + /* it is already inside existing flat mapping ranges. */ + flatSize = 0; + } + + if (flatSize == 0) + { + if (GpuBaseAddress) + { + *GpuBaseAddress = (gctUINT32) PhysBase; + } + + return gcvSTATUS_OK; + } + } + } + + /* overwrite the orignal parameters */ + PhysBase = flatBase; + physBase = (gctUINT32)flatBase; + + mtlb = _MtlbOffset(physBase); + + if (GpuBaseAddress) + { + reqVirtualBase = *GpuBaseAddress; + } + + /* + * if no partcial physical range overlap to request entire shift mapping, + * it is specific shift mapping or directly mapping by default. + */ + if (!needShiftMapping) + { + flatVirtualBase = reqVirtualBase ? reqVirtualBase : (gctUINT32)flatBase; + } + + for (i = 0; i < Mmu->gpuAddressRangeCount; i++) + { + if (_IsRangeInsected(flatVirtualBase, flatSize, + Mmu->gpuAddressRanges[i].start, Mmu->gpuAddressRanges[i].size)) + { + virtualRangeOverlapped = gcvTRUE; + } + } + + /* If gpu virtual range overlapped or gpu physical over 4G, still need entire shift mapping */ + if ((!physicalRangeOverlapped && virtualRangeOverlapped) || + PhysBase + flatSize - 1 > 0xffffffff) + { + needShiftMapping = gcvTRUE; + } + + if (needShiftMapping && !ableToShift) + { + /* + * Return without mapping any address. + * By now, only physBase physSize could run here. + */ + return gcvSTATUS_OK; + } + + specificFlatMapping = (reqVirtualBase && !virtualRangeOverlapped && !physicalRangeOverlapped); + + /************************ Setup flat mapping in dynamic range. ****************/ + if (area->mappingStart != gcvINVALID_ADDRESS && mtlb >= area->mappingStart && mtlb < area->mappingEnd) + { + /* This path is useless now, keep it 4K page size */ + + gctUINT32_PTR stlbEntry; + + stlbEntry = _StlbEntry(area, physBase); + + /* Must be aligned to page. */ + gcmkASSERT((flatSize & 0xFFF) == 0); + + for (i = 0; i < (flatSize / gcdMMU_PAGE_4K_SIZE); i++) + { + /* Flat mapping in page table. */ + _WritePageEntry(stlbEntry, _SetPage(physBase + i * gcdMMU_PAGE_4K_SIZE, 0, gcvTRUE)); +#if gcdMMU_TABLE_DUMP + gckOS_Print("%s(%d): insert MTLB[%d] STLB[%d]: %08x\n", + __FUNCTION__, __LINE__, + (physBase & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT, + ((physBase & gcdMMU_STLB_4K_MASK) >> gcdMMU_STLB_4K_SHIFT) + i, + _ReadPageEntry(stlbEntry)); +#endif + stlbEntry++; + } + +#if gcdDUMP_IN_KERNEL + { + gctPHYS_ADDR_T physical; + gctUINT32 data = _SetPage(physBase, 0, gcvTRUE) & ~0xF; + gctUINT32 step = (_SetPage(physBase + gcdMMU_PAGE_4K_SIZE, 0, gcvTRUE) & ~0xF) - data; + gctUINT32 mask = _SetPage(physBase, 0, gcvTRUE) & 0xF; + + physical = area->stlbPhysical + 4 * _AddressToIndex(area, physBase); + + gcmkDUMP(Mmu->os, + "#[mmu-stlb: flat-mapping in dynamic: 0x%08X - 0x%08X]", + physBase, physBase - 1 + flatSize); + + gcmkDUMP(Mmu->os, + "@[physical.step 0x%010llX 0x%08X 0x%08lX 0x%08X 0x%08X", + (unsigned long long)physical, data, + (unsigned long)(flatSize / gcdMMU_PAGE_4K_SIZE * sizeof(gctUINT32)), + step, mask); + } +#endif + + /* Flat mapping in map. */ + _FillFlatMappingInMap(area, _AddressToIndex(area, physBase), flatSize / gcdMMU_PAGE_4K_SIZE); + + return gcvSTATUS_OK; + } + + /************************ Setup flat mapping in non dynamic range. **************/ + { + gctBOOL mutex = gcvFALSE; + gctUINT32 physBaseExt = (gctUINT32) (PhysBase >> 32); + gctUINT32 start = physBase & ~gcdMMU_PAGE_1M_MASK; + gctUINT32 end = (gctUINT32) (physBase + flatSize - 1) & ~gcdMMU_PAGE_1M_MASK; + gctUINT32 mStart = start >> gcdMMU_MTLB_SHIFT; + gctUINT32 mEnd = end >> gcdMMU_MTLB_SHIFT; + gctUINT32 sStart = (start & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; + gctUINT32 sEnd = (end & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; + gctPHYS_ADDR_T physical; + gcsMMU_STLB_CHUNK_PTR newStlbChunk = gcvNULL; + gctUINT32 stlbIndex = 0; + gctUINT32 totalNewStlbs = 0; + gctINT32 firstMtlbEntry = -1; + gctUINT32 mtlbCurEntry; + gcsMMU_STLB_CHUNK_PTR curStlbChunk = gcvNULL; + gceFLATMAP_FLAG mapFlag = gcvFLATMAP_DIRECT; + enum + { + COLOR_NONE = 0, + COLOR_RED = 1, /* occupied entry */ + COLOR_BLUE = 2, /* empty entry */ + COLOR_MAX = COLOR_BLUE, + } lastColor = COLOR_NONE; + gctUINT32 colorNumber = 0; + + /* Grab the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); + mutex = gcvTRUE; + + if (needShiftMapping) + { + gctUINT32 mEntries; + gctUINT32 sEntries; + + mEntries = (flatSize + (1 << gcdMMU_MTLB_SHIFT) - 1) / (1 << gcdMMU_MTLB_SHIFT); + + gcmkONERROR(_GetMtlbFreeSpace(Mmu, mEntries, &mStart, &mEnd)); + + sStart = 0; + sEntries = (flatSize + gcdMMU_PAGE_1M_SIZE - 1) / gcdMMU_PAGE_1M_SIZE; + sEnd = (sEntries - 1) % gcdMMU_STLB_1M_ENTRY_NUM; + mapFlag = gcvFLATMAP_SHIFT; + } + + if (specificFlatMapping) + { + start = reqVirtualBase & ~gcdMMU_PAGE_1M_MASK; + end = (reqVirtualBase + flatSize - 1) & ~gcdMMU_PAGE_1M_MASK; + mStart = start >> gcdMMU_MTLB_SHIFT; + mEnd = end >> gcdMMU_MTLB_SHIFT; + sStart = (start & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; + sEnd = (end & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; + mapFlag = gcvFLATMAP_SHIFT; + } + + /* No matter direct mapping or shift mapping or specific mapping, store gpu virtual ranges */ + flatVirtualBase = (mStart << gcdMMU_MTLB_SHIFT) + | (sStart << gcdMMU_STLB_1M_SHIFT) + | (physBase & gcdMMU_PAGE_1M_MASK); + + /* Return GPU virtual base address if necessary */ + if (GpuBaseAddress) + { + *GpuBaseAddress = flatVirtualBase; + } + + mtlbCurEntry = mStart; + + /* find all new stlbs, part of new flat mapping range may already have stlbs*/ + while (mtlbCurEntry <= mEnd) + { + if (*(Mmu->mtlbLogical + mtlbCurEntry) == 0) + { + if (lastColor != COLOR_BLUE) + { + if (colorNumber < COLOR_MAX) + { + lastColor = COLOR_BLUE; + colorNumber++; + } + else + { + gcmkPRINT("There is a hole in new flat mapping range, which is not correct"); + } + } + + totalNewStlbs++; + if (-1 == firstMtlbEntry) + { + firstMtlbEntry = mtlbCurEntry; + } + } + else + { + if (lastColor != COLOR_RED) + { + if (colorNumber < COLOR_MAX) + { + lastColor = COLOR_RED; + colorNumber++; + } + else + { + gcmkPRINT("There is a hole in new flat mapping range, which is not correct"); + } + } + } + mtlbCurEntry++; + } + + /* Need allocate a new chunk of stlbs */ + if (totalNewStlbs) + { + gcePOOL pool = Mmu->pool; + gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS; + + gcmkONERROR( + gckOS_Allocate(Mmu->os, + sizeof(struct _gcsMMU_STLB_CHUNK), + (gctPOINTER *)&newStlbChunk)); + + newStlbChunk->mtlbEntryNum = totalNewStlbs; + newStlbChunk->next = gcvNULL; + newStlbChunk->videoMem = gcvNULL; + newStlbChunk->logical = gcvNULL; + newStlbChunk->size = gcdMMU_STLB_1M_SIZE * newStlbChunk->mtlbEntryNum; + newStlbChunk->pageCount = 0; + newStlbChunk->mtlbIndex = firstMtlbEntry; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag |= gcvALLOC_FLAG_CACHEABLE; +#endif + + allocFlag |= gcvALLOC_FLAG_4GB_ADDR; + + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + kernel, + gcdMMU_STLB_1M_SIZE, + gcvVIDMEM_TYPE_COMMAND, + allocFlag | gcvALLOC_FLAG_4K_PAGES, + &newStlbChunk->size, + &pool, + &newStlbChunk->videoMem)); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + kernel, + newStlbChunk->videoMem, + gcvFALSE, + gcvFALSE, + (gctPOINTER *)&newStlbChunk->logical)); + + gcmkONERROR(gckOS_ZeroMemory(newStlbChunk->logical, newStlbChunk->size)); + + /* Get CPU physical address. */ + gcmkONERROR(gckVIDMEM_NODE_GetPhysical( + kernel, + newStlbChunk->videoMem, + 0, + &physical)); + + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( + Mmu->os, + physical, + &physical)); + + newStlbChunk->physBase = physical; + } + + while (mStart <= mEnd) + { + gctUINT32 last = (mStart == mEnd) ? sEnd : (gcdMMU_STLB_1M_ENTRY_NUM - 1); + gctPHYS_ADDR_T stlbPhyBase; + gctUINT32_PTR stlbLogical; + + gcmkASSERT(mStart < gcdMMU_MTLB_ENTRY_NUM); + + if (*(Mmu->mtlbLogical + mStart) == 0) + { + gctUINT32 mtlbEntry; + curStlbChunk = newStlbChunk; + stlbPhyBase = curStlbChunk->physBase + (stlbIndex * gcdMMU_STLB_1M_SIZE); + stlbLogical = (gctUINT32_PTR)((gctUINT8_PTR)curStlbChunk->logical + (stlbIndex * gcdMMU_STLB_1M_SIZE)); + + physical = stlbPhyBase + /* 1MB page size */ + | (1 << 3) + /* Ignore exception */ + | (0 << 1) + /* Present */ + | (1 << 0); + + gcmkSAFECASTPHYSADDRT(mtlbEntry, physical); + + _WritePageEntry(Mmu->mtlbLogical + mStart, mtlbEntry); + +#if gcdMMU_TABLE_DUMP + gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n", + __FUNCTION__, __LINE__, + mStart, + _ReadPageEntry(Mmu->mtlbLogical + mStart)); + + gckOS_Print("%s(%d): STLB: logical:%08x -> physical:%08x\n", + __FUNCTION__, __LINE__, + stlbLogical, + stlbPhyBase); +#endif + + gcmkDUMP(Mmu->os, "#[mmu-mtlb: flat-mapping, slot: %d]", mStart); + + gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08X]", + (unsigned long long)Mmu->mtlbPhysical + mStart * 4, + Mmu->mtlbLogical[mStart], 4); + + ++stlbIndex; + } + else + { + gctUINT32 mtlbEntry = _ReadPageEntry(Mmu->mtlbLogical + mStart); + gctUINT stlbOffset; + + curStlbChunk = (gcsMMU_STLB_CHUNK_PTR)Mmu->staticSTLB; + + while (curStlbChunk) + { + if ((mStart >= curStlbChunk->mtlbIndex) && + (mStart < (curStlbChunk->mtlbIndex + curStlbChunk->mtlbEntryNum))) + { + break; + } + curStlbChunk = curStlbChunk->next; + } + gcmkASSERT(curStlbChunk); + + if (!curStlbChunk) + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + + stlbOffset = mStart - curStlbChunk->mtlbIndex; + + stlbPhyBase = curStlbChunk->physBase + (stlbOffset * gcdMMU_STLB_1M_SIZE); + stlbLogical = (gctUINT32_PTR)((gctUINT8_PTR)curStlbChunk->logical + (stlbOffset * gcdMMU_STLB_1M_SIZE)); + if (stlbPhyBase != (mtlbEntry & gcdMMU_MTLB_ENTRY_STLB_MASK)) + { + gcmkASSERT(0); + } + } + +#if gcdDUMP_IN_KERNEL + /* Start index. */ + i = sStart; + + gcmkDUMP(Mmu->os, "#[mmu-stlb: flat-mapping: 0x%08X - 0x%08X]", + start, start + (last - sStart) * gcdMMU_PAGE_1M_SIZE - 1); +#endif + + while (sStart <= last) + { + gcmkASSERT(!(start & gcdMMU_PAGE_1M_MASK)); + if (reserved) + { + /* program NOT_PRESENT | EXCEPTION for reserved entries */ + _WritePageEntry(stlbLogical + sStart, 1 << 1); + } + else + { + _WritePageEntry(stlbLogical + sStart, _SetPage(start, physBaseExt, gcvTRUE)); + } +#if gcdMMU_TABLE_DUMP + gckOS_Print("%s(%d): insert STLB[%d]: %08x\n", + __FUNCTION__, __LINE__, + sStart, + _ReadPageEntry(stlbLogical + sStart)); +#endif + /* next page. */ + start += gcdMMU_PAGE_1M_SIZE; + if (start == 0) + { + physBaseExt++; + } + sStart++; + curStlbChunk->pageCount++; + } + +#if gcdDUMP_IN_KERNEL + { + gctUINT32 data = stlbLogical[i] & ~0xF; + gctUINT32 step = (last > i) ? (stlbLogical[i + 1] - stlbLogical[i]) : 0; + gctUINT32 mask = stlbLogical[i] & 0xF; + + gcmkDUMP(Mmu->os, + "@[physical.step 0x%010llX 0x%08X 0x%08X 0x%08X 0x%08X]", + (unsigned long long)stlbPhyBase + i * 4, + data, (last - i) * 4, step, mask); + } +#endif + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + kernel, + curStlbChunk->videoMem, + 0, + curStlbChunk->logical, + curStlbChunk->size + )); + + sStart = 0; + ++mStart; + } + + gcmkASSERT(totalNewStlbs == stlbIndex); + + if (newStlbChunk) + { + /* Insert the stlbChunk into staticSTLB. */ + if (Mmu->staticSTLB == gcvNULL) + { + Mmu->staticSTLB = newStlbChunk; + } + else + { + gcmkASSERT(newStlbChunk != gcvNULL); + gcmkASSERT(newStlbChunk->next == gcvNULL); + newStlbChunk->next = Mmu->staticSTLB; + Mmu->staticSTLB = newStlbChunk; + } + } + + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); + +#if gcdENABLE_TRUST_APPLICATION + if (Mmu->hardware->options.secureMode == gcvSECURE_IN_TA) + { + gckKERNEL_SecurityMapMemory(Mmu->hardware->kernel, gcvNULL, physBase, flatSize / gcdMMU_PAGE_4K_SIZE, &physBase); + } +#endif + + /* Store the gpu physical ranges */ + Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].start = flatBase; + Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].end = flatBase + flatSize - 1; + Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].size = flatSize; + Mmu->gpuPhysicalRanges[Mmu->gpuPhysicalRangeCount].flag = mapFlag; + Mmu->gpuPhysicalRangeCount++; + + gcmkASSERT(Mmu->gpuPhysicalRangeCount <= gcdMAX_FLAT_MAPPING_COUNT); + + /* Store the gpu virtual ranges */ + Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].start = flatVirtualBase; + Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].end = flatVirtualBase + flatSize - 1; + Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].size = flatSize; + Mmu->gpuAddressRanges[Mmu->gpuAddressRangeCount].flag = mapFlag; + Mmu->gpuAddressRangeCount++; + + gcmkASSERT(Mmu->gpuAddressRangeCount <= gcdMAX_FLAT_MAPPING_COUNT); + + return gcvSTATUS_OK; +OnError: + /* Roll back the allocation. + ** We don't need roll back mtlb programming as gckmONERROR + ** is only used during allocation time. + */ + if (newStlbChunk) + { + if (newStlbChunk->videoMem) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + kernel, + newStlbChunk->videoMem + )); + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, newStlbChunk)); + } + if (mutex) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); + } + return status; + } +} + +static gceSTATUS +_SetupAddressArea( + IN gckOS Os, + IN gcsADDRESS_AREA_PTR Area, + IN gctUINT32 NumMTLBEntries + ) +{ + gceSTATUS status; + gctUINT32_PTR map; + gctUINT32 stlbSize = (Area->areaType == gcvAREA_TYPE_1M) + ? gcdMMU_STLB_1M_SIZE : gcdMMU_STLB_4K_SIZE; + + gcmkHEADER(); + Area->stlbSize = NumMTLBEntries * stlbSize; + + gcmkSAFECASTSIZET(Area->stlbEntries, Area->stlbSize / gcmSIZEOF(gctUINT32)); + + gcmkONERROR(gckOS_Allocate(Os, Area->stlbSize, (void **)&Area->mapLogical)); + + /* Initialization. */ + map = Area->mapLogical; + map[0] = (Area->stlbEntries << 8) | gcvMMU_FREE; + map[1] = ~0U; + Area->heapList = 0; + Area->freeNodes = gcvFALSE; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_ConstructDynamicStlb( + IN gckMMU Mmu, + IN gcsADDRESS_AREA_PTR Area, + IN gctUINT32 NumEntries + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctBOOL acquired = gcvFALSE; + gckKERNEL kernel = Mmu->hardware->kernel; + gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS; + gcePOOL pool = Mmu->pool; + gctUINT32 address; + gctUINT32 mtlbEntry; + gctUINT32 i; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag |= gcvALLOC_FLAG_CACHEABLE; +#endif + + allocFlag |= gcvALLOC_FLAG_4GB_ADDR; + + /* Construct Slave TLB. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + allocFlag | gcvALLOC_FLAG_4K_PAGES, + &Area->stlbSize, + &pool, + &Area->stlbVideoMem)); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + kernel, + Area->stlbVideoMem, + gcvFALSE, + gcvFALSE, + (gctPOINTER *)&Area->stlbLogical)); + +#if gcdUSE_MMU_EXCEPTION + gcmkONERROR(_FillPageTable(Area->stlbLogical, + Area->stlbEntries, + /* Enable exception */ + 1 << 1)); +#else + /* Invalidate all entries. */ + gcmkONERROR(gckOS_ZeroMemory(Area->stlbLogical, + Area->stlbSize)); +#endif + + /* Get stlb table physical. */ + gcmkONERROR(gckVIDMEM_NODE_GetPhysical( + kernel, + Area->stlbVideoMem, + 0, + &Area->stlbPhysical)); + + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Mmu->os, + Area->stlbPhysical, + &Area->stlbPhysical)); + + if (Area->areaType == gcvAREA_TYPE_1M) + { + gcmkDUMP(Mmu->os, "#[mmu: 1M page size dynamic space: 0x%08X - 0x%08X]", + (Area->mappingStart << gcdMMU_MTLB_SHIFT), + (Area->mappingEnd << gcdMMU_MTLB_SHIFT) - 1); + } + else + { + gcmkDUMP(Mmu->os, "#[mmu: 4K page size dynamic space: 0x%08X - 0x%08X]", + (Area->mappingStart << gcdMMU_MTLB_SHIFT), + (Area->mappingEnd << gcdMMU_MTLB_SHIFT) - 1); + } + + gcmkDUMP(Mmu->os, "#[mmu-stlb]"); + + gcmkDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 0x%08lX]", + (unsigned long long)Area->stlbPhysical, + Area->stlbLogical[0], + (unsigned long)Area->stlbSize); + + gcmkSAFECASTPHYSADDRT(address, Area->stlbPhysical); + + /* Grab the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Map to Master TLB. */ + for (i = Area->mappingStart; + i < Area->mappingStart + NumEntries; + i++) + { + if (Area->areaType == gcvAREA_TYPE_1M) + { + mtlbEntry = address + /* 1M page size */ + | (1 << 3) + /*Ignore exception */ + | (0 << 1) + /* Present */ + | (1 << 0); + + address += gcdMMU_STLB_1M_SIZE; + } + else + { + mtlbEntry = address + /* 4KB page size */ + | (0 << 2) + /*Ignore exception */ + | (0 << 1) + /* Present */ + | (1 << 0); + + address += gcdMMU_STLB_4K_SIZE; + } + + _WritePageEntry(Mmu->mtlbLogical + i, mtlbEntry); + +#if gcdMMU_TABLE_DUMP + gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n", + __FUNCTION__, __LINE__, + i, + _ReadPageEntry(Mmu->mtlbLogical + i)); +#endif + } + + gcmkDUMP(Mmu->os, "#[mmu-mtlb: slot: %d - %d]", + Area->mappingStart, Area->mappingEnd - 1); + +#if gcdDUMP_IN_KERNEL + { + gctUINT32 data = Mmu->mtlbLogical[Area->mappingStart] & ~0x3F; + gctUINT32 step = 0; + gctUINT32 mask = Mmu->mtlbLogical[Area->mappingStart] & 0x3F; + + if (NumEntries > 1) + { + step = Mmu->mtlbLogical[Area->mappingStart + 1] + - Mmu->mtlbLogical[Area->mappingStart]; + } + + gcmkDUMP(Mmu->os, + "@[physical.step 0x%010llX 0x%08X 0x%08X 0x%08X 0x%08X]", + (unsigned long long)(Mmu->mtlbPhysical + Area->mappingStart * 4), + data, NumEntries * 4, step, mask); + } +#endif + +OnError: + if (acquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); + } + + return status; +} + +gceSTATUS +gckMMU_SetupDynamicSpace( + IN gckMMU Mmu + ) +{ + gceSTATUS status; + gcsFreeSpaceNode_PTR nodeArray = gcvNULL; + gctINT i, nodeArraySize = 0; + gctINT numEntries = 0; + gckKERNEL kernel = Mmu->hardware->kernel; + gcsADDRESS_AREA_PTR area4K = &Mmu->dynamicArea4K; +#if gcdENABLE_GPU_1M_PAGE + gcsADDRESS_AREA_PTR area1M = &Mmu->dynamicArea1M; + gctINT numEntries1M; +#endif + + /* Find all the free address space. */ + gcmkONERROR(_CollectFreeSpace(Mmu, &nodeArray, &nodeArraySize)); + + for (i = 0; i < nodeArraySize; i++) + { + if (nodeArray[i].entries > numEntries) + { + area4K->mappingStart = nodeArray[i].start; + numEntries = nodeArray[i].entries; + area4K->mappingEnd = area4K->mappingStart + numEntries - 1; + } + } + + gckOS_Free(Mmu->os, (gctPOINTER)nodeArray); + +#if gcdENABLE_TRUST_APPLICATION + if (gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_SECURITY) == gcvSTATUS_TRUE) + { + /* Setup secure address area when needed. */ + gctUINT32 secureAreaSize = gcdMMU_SECURE_AREA_SIZE; + gcsADDRESS_AREA_PTR secureArea = &Mmu->secureArea; + + gcmkASSERT(numEntries > (gctINT)secureAreaSize); + + secureArea->mappingStart = area4K->mappingStart + + (numEntries - secureAreaSize); + + gcmkONERROR(_SetupAddressArea(Mmu->os, secureArea, secureAreaSize)); + + numEntries -= secureAreaSize; + area4K->mappingEnd -= secureAreaSize; + } +#endif + +#if gcdENABLE_GPU_1M_PAGE + numEntries1M = numEntries >> 1; + area1M->mappingStart = area4K->mappingStart + (numEntries - numEntries1M); + area1M->mappingEnd = area1M->mappingStart + numEntries1M - 1; + area1M->areaType = gcvAREA_TYPE_1M; + numEntries -= numEntries1M; + area4K->mappingEnd -= numEntries1M; + + gcmkONERROR(_SetupAddressArea(Mmu->os, area1M, numEntries1M)); + + gcmkONERROR(_ConstructDynamicStlb(Mmu, area1M, numEntries1M)); +#endif + + area4K->areaType = gcvAREA_TYPE_4K; + + /* Setup normal address area. */ + gcmkONERROR(_SetupAddressArea(Mmu->os, area4K, numEntries)); + + gcmkONERROR(_ConstructDynamicStlb(Mmu, area4K, numEntries)); + + return gcvSTATUS_OK; + +OnError: +#if gcdENABLE_GPU_1M_PAGE + if (area1M->mapLogical) + { + gcmkVERIFY_OK( + gckOS_Free(Mmu->os, (gctPOINTER) area1M->mapLogical)); + } + + if (area1M->stlbVideoMem) + { + gcmkVERIFY_OK( + gckVIDMEM_NODE_Dereference(kernel, + area1M->stlbVideoMem)); + } +#endif + + if (area4K->mapLogical) + { + gcmkVERIFY_OK( + gckOS_Free(Mmu->os, (gctPOINTER) area4K->mapLogical)); + } + + if (area4K->stlbVideoMem) + { + gcmkVERIFY_OK( + gckVIDMEM_NODE_Dereference(kernel, + area4K->stlbVideoMem)); + } + + return status; +} + +gctUINT32 +_GetPageCountOfUsedNode( + gctUINT32_PTR Node + ) +{ + gctUINT32 count; + + count = gcmENTRY_COUNT(*Node); + + if ((count << 8) == (~((1U<<8)-1))) + { + count = 1; + } + + return count; +} + +static gcsADDRESS_AREA_PTR +_GetProcessArea( + IN gckMMU Mmu, + IN gcePAGE_TYPE PageType, + IN gctBOOL Secure + ) +{ +#if gcdENABLE_TRUST_APPLICATION + if (Secure == gcvTRUE) + { + return &Mmu->secureArea; + } +#endif + + if (PageType == gcvPAGE_TYPE_1M) + { + return &Mmu->dynamicArea1M; + } + else + { + return &Mmu->dynamicArea4K; + } +} + +/******************************************************************************* +** +** _Construct +** +** Construct a new gckMMU object. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctSIZE_T MmuSize +** Number of bytes for the page table. +** +** OUTPUT: +** +** gckMMU * Mmu +** Pointer to a variable that receives the gckMMU object pointer. +*/ +gceSTATUS +_Construct( + IN gckKERNEL Kernel, + IN gctSIZE_T MmuSize, + OUT gckMMU * Mmu + ) +{ + gckOS os; + gckHARDWARE hardware; + gceSTATUS status; + gckMMU mmu = gcvNULL; + gctUINT32_PTR map; + gctPOINTER pointer = gcvNULL; + gctPHYS_ADDR_T physBase; + gctSIZE_T physSize; + gctPHYS_ADDR_T contiguousBase; + gctSIZE_T contiguousSize = 0; + gctPHYS_ADDR_T externalBase; + gctSIZE_T externalSize = 0; + gctUINT32 gpuAddress; + gctPHYS_ADDR_T gpuPhysical; + gcsADDRESS_AREA_PTR area = gcvNULL; + gcePOOL pool; + gctUINT64 data; + gctUINT32 allocFlag = gcvALLOC_FLAG_CONTIGUOUS; + gctUINT64 mmuEnabled; + + gcmkHEADER_ARG("Kernel=0x%x MmuSize=%lu", Kernel, MmuSize); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(MmuSize > 0); + gcmkVERIFY_ARGUMENT(Mmu != gcvNULL); + + /* Extract the gckOS object pointer. */ + os = Kernel->os; + gcmkVERIFY_OBJECT(os, gcvOBJ_OS); + + /* Extract the gckHARDWARE object pointer. */ + hardware = Kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Allocate memory for the gckMMU object. */ + gcmkONERROR(gckOS_Allocate(os, sizeof(struct _gckMMU), &pointer)); + + gckOS_ZeroMemory(pointer, sizeof(struct _gckMMU)); + + mmu = pointer; + + /* Initialize the gckMMU object. */ + mmu->object.type = gcvOBJ_MMU; + mmu->os = os; + mmu->hardware = hardware; + mmu->pageTableMutex = gcvNULL; + mmu->mtlbLogical = gcvNULL; + mmu->staticSTLB = gcvNULL; + mmu->enabled = gcvFALSE; + + mmu->dynamicAreaSetuped = gcvFALSE; + mmu->pool = _GetPageTablePool(mmu->os); + gcsLIST_Init(&mmu->hardwareList); + + /* Use 4K page size for MMU version 0. */ + area = &mmu->dynamicArea4K; + area->mapLogical = gcvNULL; + area->stlbLogical = gcvNULL; + + /* Create the page table mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &mmu->pageTableMutex)); + + gcmkONERROR(gckOS_QueryOption(os, "mmu", &mmuEnabled)); + + if (hardware->mmuVersion == 0) + { + area->stlbSize = MmuSize; + + /* Construct address space management table. */ + gcmkONERROR(gckOS_Allocate(mmu->os, + area->stlbSize, + &pointer)); + + area->mapLogical = pointer; + + pool = mmu->pool; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag |= gcvALLOC_FLAG_CACHEABLE; +#endif + + /* Construct page table read by GPU. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + Kernel, + 4096, + gcvVIDMEM_TYPE_COMMAND, + allocFlag, + &area->stlbSize, + &pool, + &area->stlbVideoMem)); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + Kernel, + area->stlbVideoMem, + gcvFALSE, + gcvFALSE, + &pointer)); + + area->stlbLogical = pointer; + + /* Get CPU physical address. */ + gcmkONERROR(gckVIDMEM_NODE_GetPhysical( + Kernel, + area->stlbVideoMem, + 0, + &area->stlbPhysical)); + + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(mmu->os, + area->stlbPhysical, + &area->stlbPhysical)); + + /* Compute number of entries in page table. */ + gcmkSAFECASTSIZET(area->stlbEntries, area->stlbSize / sizeof(gctUINT32)); + + /* Mark all pages as free. */ + map = area->mapLogical; + + _FillPageTable(area->stlbLogical, area->stlbEntries, mmu->safeAddress); + + gcmkDUMP(mmu->os, + "#[mmu0: fill with safe address]"); + + gcmkDUMP(mmu->os, + "@[physical.fill 0x%010llX 0x%08X 0x%08lX]", + (unsigned long long)area->stlbPhysical, + area->stlbLogical[0], + (unsigned long)area->stlbSize); + + map[0] = (area->stlbEntries << 8) | gcvMMU_FREE; + map[1] = ~0U; + area->heapList = 0; + area->freeNodes = gcvFALSE; + + status = gckOS_QueryOption(mmu->os, "contiguousBase", &contiguousBase); + + if (gcmIS_SUCCESS(status)) + { + status = gckOS_QueryOption(mmu->os, "contiguousSize", &data); + contiguousSize = (gctSIZE_T)data; + } + + if (gcmIS_SUCCESS(status) && contiguousSize) + { + /* Convert to GPU address. */ + mmu->contiguousBaseAddress = (gctUINT32)(contiguousBase - Kernel->hardware->baseAddress); + } + } + else + { + mmu->mtlbSize = gcdMMU_MTLB_SIZE; + + pool = mmu->pool; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag |= gcvALLOC_FLAG_CACHEABLE; +#endif + + allocFlag |= gcvALLOC_FLAG_4GB_ADDR; + + /* 1K mode is 1024 byte aligned. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + Kernel, + 1024, + gcvVIDMEM_TYPE_COMMAND, + allocFlag | gcvALLOC_FLAG_4K_PAGES, + &mmu->mtlbSize, + &pool, + &mmu->mtlbVideoMem)); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + Kernel, + mmu->mtlbVideoMem, + gcvFALSE, + gcvFALSE, + &pointer)); + + mmu->mtlbLogical = pointer; + + mmu->dynamicArea4K.mappingStart = gcvINVALID_ADDRESS; + mmu->dynamicArea1M.mappingStart = gcvINVALID_ADDRESS; + + /* Get mtlb table physical. */ + gcmkONERROR(gckVIDMEM_NODE_GetPhysical( + Kernel, + mmu->mtlbVideoMem, + 0, + &mmu->mtlbPhysical)); + + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( + mmu->os, + mmu->mtlbPhysical, + &mmu->mtlbPhysical)); + + /* Invalid all the entries. */ + gcmkONERROR( + gckOS_ZeroMemory(pointer, mmu->mtlbSize)); + + gcmkONERROR( + gckOS_QueryOption(mmu->os, "physBase", &physBase)); + + gcmkONERROR( + gckOS_QueryOption(mmu->os, "physSize", &data)); + physSize = (gctSIZE_T)data; + + gcmkONERROR( + gckOS_CPUPhysicalToGPUPhysical(mmu->os, physBase, &gpuPhysical)); + + gcmkSAFECASTPHYSADDRT(gpuAddress, gpuPhysical); + + if (physSize) + { + /* Setup user specified flat mapping. */ + gcmkONERROR(_FillFlatMapping(mmu, gpuAddress, physSize, gcvFALSE, gcvFALSE, gcvNULL)); + } + +#if !(0 || gcdCAPTURE_ONLY_MODE) + if (!_ReadPageEntry(mmu->mtlbLogical + 0)) + { + gctUINT32 mtlbEntry; + /* + * Reserved the first mtlb. + * 1MB page size, Ingore exception, Not Present. + */ + mtlbEntry = (1 << 3) + | (0 << 1) + | (0 << 0); + + _WritePageEntry(mmu->mtlbLogical + 0, mtlbEntry); + + gcmkDUMP(mmu->os, "#[mmu-mtlb: reserved 16M space, slot: 0]"); + gcmkDUMP(mmu->os, + "@[physical.fill 0x%010llX 0x%08X 0x%08X]", + (unsigned long long)mmu->mtlbPhysical, + mmu->mtlbLogical[0], 4); + + /* Store the gpu virtual ranges */ + mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].start = 0; + mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].end = (16 << 20) - 1; + mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].size = (16 << 20); + mmu->gpuAddressRanges[mmu->gpuAddressRangeCount].flag = gcvFLATMAP_DIRECT; + mmu->gpuAddressRangeCount++; + } +#endif + + status = gckOS_QueryOption(mmu->os, "contiguousBase", &contiguousBase); + + if (gcmIS_SUCCESS(status)) + { + status = gckOS_QueryOption(mmu->os, "contiguousSize", &data); + contiguousSize = (gctSIZE_T)data; + } + + if (gcmIS_SUCCESS(status) && contiguousSize) + { + gctUINT64 gpuContiguousBase; + gctUINT32 contiguousBaseAddress = 0; + + gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(mmu->os, contiguousBase, &gpuContiguousBase)); + + /* Setup flat mapping for reserved memory (VIDMEM). */ + gcmkONERROR(_FillFlatMapping(mmu, gpuContiguousBase, contiguousSize, gcvFALSE, gcvTRUE, &contiguousBaseAddress)); + + if (mmuEnabled) + { + mmu->contiguousBaseAddress = contiguousBaseAddress; + } + else + { + gcmkSAFECASTPHYSADDRT(mmu->contiguousBaseAddress, gpuContiguousBase); + } + } + + status = gckOS_QueryOption(mmu->os, "externalBase", &externalBase); + + if (gcmIS_SUCCESS(status)) + { + status = gckOS_QueryOption(mmu->os, "externalSize", &data); + externalSize = (gctSIZE_T)data; + } + + if (gcmIS_SUCCESS(status) && externalSize) + { + gctUINT64 gpuExternalBase; + gctUINT32 externalBaseAddress = 0; + + gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical(mmu->os, externalBase, &gpuExternalBase)); + + /* Setup flat mapping for external memory. */ + gcmkONERROR(_FillFlatMapping(mmu, gpuExternalBase, externalSize, gcvFALSE, gcvTRUE, &externalBaseAddress)); + + mmu->externalBaseAddress = externalBaseAddress; + } + } + + /* A 64 byte for safe address, we use 256 here. */ + mmu->safePageSize = 256; + + pool = mmu->pool; + + /* Allocate safe page from video memory. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + Kernel, + 256, + gcvVIDMEM_TYPE_COMMAND, + gcvALLOC_FLAG_CONTIGUOUS | gcvALLOC_FLAG_4K_PAGES | gcvALLOC_FLAG_4GB_ADDR, + &mmu->safePageSize, + &pool, + &mmu->safePageVideoMem + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + Kernel, + mmu->safePageVideoMem, + gcvFALSE, + gcvFALSE, + &mmu->safePageLogical + )); + + /* Get CPU physical address. */ + gcmkONERROR(gckVIDMEM_NODE_GetPhysical( + Kernel, + mmu->safePageVideoMem, + 0, + &mmu->safePagePhysical + )); + + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( + os, + mmu->safePagePhysical, + &mmu->safePagePhysical + )); + + gcmkSAFECASTPHYSADDRT(mmu->safeAddress, mmu->safePagePhysical); + + gckOS_ZeroMemory(mmu->safePageLogical, mmu->safePageSize); + + gcmkDUMP(mmu->os, "#[safe page]"); + gcmkDUMP(mmu->os, + "@[physical.fill 0x%010llX 0x%08X 0x%08lX]", + (unsigned long long)mmu->safePagePhysical, + 0, + (unsigned long)mmu->safePageSize); + + gcmkDUMP_BUFFER( + mmu->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + mmu->safePageLogical, + mmu->safeAddress, + mmu->safePageSize + ); + + gcmkONERROR(gckQUEUE_Allocate(os, &mmu->recentFreedAddresses, 16)); + + mmu->sRAMMapped = gcvFALSE; + + /* Return the gckMMU object pointer. */ + *Mmu = mmu; + + /* Success. */ + gcmkFOOTER_ARG("*Mmu=0x%x", *Mmu); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (mmu != gcvNULL) + { + if (area != gcvNULL && area->mapLogical != gcvNULL) + { + gcmkVERIFY_OK( + gckOS_Free(os, (gctPOINTER) area->mapLogical)); + + gcmkVERIFY_OK( + gckVIDMEM_NODE_Dereference(Kernel, + area->stlbVideoMem)); + } + + if (mmu->mtlbLogical != gcvNULL) + { + gcmkVERIFY_OK( + gckVIDMEM_NODE_Dereference(Kernel, + mmu->mtlbVideoMem)); + } + + if (mmu->pageTableMutex != gcvNULL) + { + /* Delete the mutex. */ + gcmkVERIFY_OK( + gckOS_DeleteMutex(os, mmu->pageTableMutex)); + } + + gcmkVERIFY_OK(gckQUEUE_Free(os, &mmu->recentFreedAddresses)); + + /* Mark the gckMMU object as unknown. */ + mmu->object.type = gcvOBJ_UNKNOWN; + + /* Free the allocates memory. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, mmu)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_FreeAddressArea( + gckKERNEL Kernel, + gcsADDRESS_AREA * Area + ) +{ + gceSTATUS status = gcvSTATUS_OK; + + if (Area->mapLogical != gcvNULL) + { + gcmkVERIFY_OK( + gckOS_Free(Kernel->os, (gctPOINTER) Area->mapLogical)); + } + + if (Area->stlbLogical != gcvNULL) + { + /* Free page table. */ + gcmkVERIFY_OK( + gckVIDMEM_NODE_Dereference(Kernel, + Area->stlbVideoMem)); + } + + return status; +} + +/******************************************************************************* +** +** _Destroy +** +** Destroy a gckMMU object. +** +** INPUT: +** +** gckMMU Mmu +** Pointer to an gckMMU object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +_Destroy( + IN gckMMU Mmu + ) +{ + gckKERNEL kernel = Mmu->hardware->kernel; + + gcmkHEADER_ARG("Mmu=0x%x", Mmu); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); + + while (Mmu->staticSTLB != gcvNULL) + { + gcsMMU_STLB_CHUNK_PTR pre = Mmu->staticSTLB; + Mmu->staticSTLB = pre->next; + + if (pre->videoMem) + { + gcmkVERIFY_OK( + gckVIDMEM_NODE_Dereference(kernel, + pre->videoMem)); + } + + if (pre->mtlbEntryNum != 0) + { + gctUINT i; + for (i = 0; i < pre->mtlbEntryNum; ++i) + { + _WritePageEntry(Mmu->mtlbLogical + pre->mtlbIndex + i, 0); +#if gcdMMU_TABLE_DUMP + gckOS_Print("%s(%d): clean MTLB[%d]\n", + __FUNCTION__, __LINE__, + pre->mtlbIndex + i); +#endif + } + + gcmkDUMP(Mmu->os, + "#[mmu-mtlb: clean up slot: %d - %d]", + pre->mtlbIndex, + pre->mtlbIndex + pre->mtlbEntryNum - 1); + + gcmkDUMP(Mmu->os, + "@[physical.fill 0x%010llX 0x%08X 0x%08lX]", + (unsigned long long)(Mmu->mtlbPhysical + pre->mtlbIndex * 4), + Mmu->mtlbLogical[pre->mtlbIndex], + (unsigned long)(pre->mtlbEntryNum * 4)); + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, pre)); + } + + if (Mmu->hardware->mmuVersion != 0) + { + gcmkVERIFY_OK( + gckVIDMEM_NODE_Dereference(kernel, + Mmu->mtlbVideoMem)); + } + + /* Free address area. */ + gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->dynamicArea4K)); +#if gcdENABLE_GPU_1M_PAGE + gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->dynamicArea1M)); +#endif + gcmkVERIFY_OK(_FreeAddressArea(kernel, &Mmu->secureArea)); + + /* Delete the page table mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Mmu->os, Mmu->pageTableMutex)); + + if (Mmu->safePageLogical != gcvNULL) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + kernel, + Mmu->safePageVideoMem + )); + } + + gcmkVERIFY_OK(gckQUEUE_Free(Mmu->os, &Mmu->recentFreedAddresses)); + + /* Mark the gckMMU object as unknown. */ + Mmu->object.type = gcvOBJ_UNKNOWN; + + /* Free the gckMMU object. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, Mmu)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** _AdjstIndex +** +** Adjust the index from which we search for a usable node to make sure +** index allocated is greater than Start. +*/ +gceSTATUS +_AdjustIndex( + IN gckMMU Mmu, + IN gctUINT32 Index, + IN gctUINT32 PageCount, + IN gctUINT32 Start, + OUT gctUINT32 * IndexAdjusted + ) +{ + gceSTATUS status; + gctUINT32 index = Index; + gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K; + gctUINT32_PTR map = area->mapLogical; + + gcmkHEADER(); + + for (; index < area->stlbEntries;) + { + gctUINT32 result = 0; + gctUINT32 nodeSize = 0; + + if (index >= Start) + { + break; + } + + switch (gcmENTRY_TYPE(map[index])) + { + case gcvMMU_SINGLE: + nodeSize = 1; + break; + + case gcvMMU_FREE: + nodeSize = map[index] >> 8; + break; + + default: + gcmkFATAL("MMU table correcupted at index %u!", index); + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + + if (nodeSize > PageCount) + { + result = index + (nodeSize - PageCount); + + if (result >= Start) + { + break; + } + } + + switch (gcmENTRY_TYPE(map[index])) + { + case gcvMMU_SINGLE: + index = map[index] >> 8; + break; + + case gcvMMU_FREE: + index = map[index + 1]; + break; + + default: + gcmkFATAL("MMU table correcupted at index %u!", index); + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + } + + *IndexAdjusted = index; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckMMU_Construct( + IN gckKERNEL Kernel, + IN gctSIZE_T MmuSize, + OUT gckMMU * Mmu + ) +{ +#if gcdSHARED_PAGETABLE + gceSTATUS status; + gctPOINTER pointer; + + gcmkHEADER_ARG("Kernel=0x%08x", Kernel); + + if (sharedPageTable == gcvNULL) + { + gcmkONERROR( + gckOS_Allocate(Kernel->os, + sizeof(struct _gcsSharedPageTable), + &pointer)); + sharedPageTable = pointer; + + gcmkONERROR( + gckOS_ZeroMemory(sharedPageTable, + sizeof(struct _gcsSharedPageTable))); + + gcmkONERROR(_Construct(Kernel, MmuSize, &sharedPageTable->mmu)); + } + + *Mmu = sharedPageTable->mmu; + + sharedPageTable->hardwares[sharedPageTable->reference] = Kernel->hardware; + + sharedPageTable->reference++; + + gcmkFOOTER_ARG("sharedPageTable->reference=%lu", sharedPageTable->reference); + return gcvSTATUS_OK; + +OnError: + if (sharedPageTable) + { + if (sharedPageTable->mmu) + { + gcmkVERIFY_OK(gckMMU_Destroy(sharedPageTable->mmu)); + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, sharedPageTable)); + } + + gcmkFOOTER(); + return status; +#else + return _Construct(Kernel, MmuSize, Mmu); +#endif +} + +gceSTATUS +gckMMU_Destroy( + IN gckMMU Mmu + ) +{ +#if gcdSHARED_PAGETABLE + gckOS os = Mmu->os; + + sharedPageTable->reference--; + + if (sharedPageTable->reference == 0) + { + if (sharedPageTable->mmu) + { + gcmkVERIFY_OK(_Destroy(Mmu)); + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, sharedPageTable)); + } + + return gcvSTATUS_OK; +#else + return _Destroy(Mmu); +#endif +} + +/******************************************************************************* +** +** gckMMU_AllocatePages +** +** Allocate pages inside the page table. +** +** INPUT: +** +** gckMMU Mmu +** Pointer to an gckMMU object. +** +** gctSIZE_T PageCount +** Number of pages to allocate. +** +** OUTPUT: +** +** gctPOINTER * PageTable +** Pointer to a variable that receives the base address of the page +** table. +** +** gctUINT32 * Address +** Pointer to a variable that receives the hardware specific address. +*/ +gceSTATUS +_AllocatePages( + IN gckMMU Mmu, + IN gctSIZE_T PageCount, + IN gceVIDMEM_TYPE Type, + IN gcePAGE_TYPE PageType, + IN gctBOOL Secure, + OUT gctPOINTER * PageTable, + OUT gctUINT32 * Address + ) +{ + gceSTATUS status; + gctBOOL mutex = gcvFALSE; + gctUINT32 index = 0, previous = ~0U, left; + gctUINT32_PTR map; + gctBOOL gotIt; + gctUINT32 address; + gctUINT32 pageCount; + gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, Secure); + + gcmkHEADER_ARG("Mmu=0x%x PageCount=%lu", Mmu, PageCount); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); + gcmkVERIFY_ARGUMENT(PageCount > 0); + gcmkVERIFY_ARGUMENT(PageTable != gcvNULL); + + if (PageCount > area->stlbEntries) + { + /* Not enough pages avaiable. */ + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + + gcmkSAFECASTSIZET(pageCount, PageCount); + +#if gcdBOUNDARY_CHECK + /* Extra pages as bounary. */ + pageCount += gcdBOUNDARY_CHECK * 2; +#endif + + /* Grab the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); + mutex = gcvTRUE; + + /* Cast pointer to page table. */ + for (map = area->mapLogical, gotIt = gcvFALSE; !gotIt;) + { + index = area->heapList; + + if ((Mmu->hardware->mmuVersion == 0) && + (Type == gcvVIDMEM_TYPE_VERTEX_BUFFER)) + { + gcmkONERROR(_AdjustIndex( + Mmu, + index, + pageCount, + gcdVERTEX_START / gcmSIZEOF(gctUINT32), + &index + )); + } + + /* Walk the heap list. */ + for (; !gotIt && (index < area->stlbEntries);) + { + /* Check the node type. */ + switch (gcmENTRY_TYPE(map[index])) + { + case gcvMMU_SINGLE: + /* Single odes are valid if we only need 1 page. */ + if (pageCount == 1) + { + gotIt = gcvTRUE; + } + else + { + /* Move to next node. */ + previous = index; + index = map[index] >> 8; + } + break; + + case gcvMMU_FREE: + /* Test if the node has enough space. */ + if (pageCount <= (map[index] >> 8)) + { + gotIt = gcvTRUE; + } + else + { + /* Move to next node. */ + previous = index; + index = map[index + 1]; + } + break; + + default: + gcmkFATAL("MMU table correcupted at index %u!", index); + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + } + + /* Test if we are out of memory. */ + if (index >= area->stlbEntries) + { + if (area->freeNodes) + { + /* Time to move out the trash! */ + gcmkONERROR(_Collect(area)); + + /* We are going to search from start, so reset previous to start. */ + previous = ~0U; + } + else + { + /* Out of resources. */ + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + } + } + + switch (gcmENTRY_TYPE(map[index])) + { + case gcvMMU_SINGLE: + /* Unlink single node from free list. */ + gcmkONERROR( + _Link(area, previous, map[index] >> 8)); + break; + + case gcvMMU_FREE: + /* Check how many pages will be left. */ + left = (map[index] >> 8) - pageCount; + switch (left) + { + case 0: + /* The entire node is consumed, just unlink it. */ + gcmkONERROR( + _Link(area, previous, map[index + 1])); + break; + + case 1: + /* One page will remain. Convert the node to a single node and + ** advance the index. */ + map[index] = (map[index + 1] << 8) | gcvMMU_SINGLE; + index ++; + break; + + default: + /* Enough pages remain for a new node. However, we will just adjust + ** the size of the current node and advance the index. */ + map[index] = (left << 8) | gcvMMU_FREE; + index += left; + break; + } + break; + } + + /* Mark node as used. */ + gcmkONERROR(_FillMap(&map[index], pageCount, gcvMMU_USED)); + +#if gcdBOUNDARY_CHECK + index += gcdBOUNDARY_CHECK; +#endif + + /* Record pageCount of allocated node at the beginning of node. */ + if (pageCount == 1) + { + map[index] = (~((1U<<8)-1)) | gcvMMU_USED; + } + else + { + map[index] = (pageCount << 8) | gcvMMU_USED; + } + + if (area->stlbLogical != gcvNULL) + { + /* Return pointer to page table. */ + *PageTable = &area->stlbLogical[index]; + } + else + { + /* Page table for secure area is handled in trust application. */ + *PageTable = gcvNULL; + } + + /* Build virtual address. */ + if (Mmu->hardware->mmuVersion == 0) + { + gcmkONERROR( + gckHARDWARE_BuildVirtualAddress(Mmu->hardware, index, 0, &address)); + } + else + { + gctUINT32 num = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM; + gctUINT32 shift = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT; + gctUINT32 masterOffset = index / num + area->mappingStart; + gctUINT32 slaveOffset = index % num; + + address = (masterOffset << gcdMMU_MTLB_SHIFT) + | (slaveOffset << shift); + } + + if (Address != gcvNULL) + { + *Address = address; + } + + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); + + /* Success. */ + gcmkFOOTER_ARG("*PageTable=0x%x *Address=%08x", + *PageTable, gcmOPT_VALUE(Address)); + return gcvSTATUS_OK; + +OnError: + + if (mutex) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckMMU_FreePages +** +** Free pages inside the page table. +** +** INPUT: +** +** gckMMU Mmu +** Pointer to an gckMMU object. +** +** gctPOINTER PageTable +** Base address of the page table to free. +** +** gctSIZE_T PageCount +** Number of pages to free. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +_FreePages( + IN gckMMU Mmu, + IN gctBOOL Secure, + IN gcePAGE_TYPE PageType, + IN gctUINT32 Address, + IN gctPOINTER PageTable, + IN gctSIZE_T PageCount + ) +{ + gctUINT32 index; + gctUINT32_PTR node; + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gctUINT32 pageCount; + gcuQUEUEDATA data; + gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, gcvFALSE); + gctUINT32 pageSize = (PageType == gcvPAGE_TYPE_1M) + ? gcdMMU_PAGE_1M_SIZE : gcdMMU_PAGE_4K_SIZE; + + gcmkHEADER_ARG("Mmu=0x%x PageTable=0x%x PageCount=%lu", + Mmu, PageTable, PageCount); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); + gcmkVERIFY_ARGUMENT(PageCount > 0); + + gcmkSAFECASTSIZET(pageCount, PageCount); + +#if gcdBOUNDARY_CHECK + pageCount += gcdBOUNDARY_CHECK * 2; +#endif + + /* Get the node by index. */ + index = (gctUINT32)((gctUINT32_PTR)PageTable - area->stlbLogical); + + node = area->mapLogical + index; + + if (pageCount != _GetPageCountOfUsedNode(node)) + { + gcmkONERROR(gcvSTATUS_INVALID_REQUEST); + } + +#if gcdBOUNDARY_CHECK + node -= gcdBOUNDARY_CHECK; +#endif + + gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); + acquired = gcvTRUE; + + if (Mmu->hardware->mmuVersion == 0) + { + _FillPageTable(PageTable, pageCount, Mmu->safeAddress); + } + + if (pageCount == 1) + { + /* Single page node. */ + node[0] = (~((1U<<8)-1)) | gcvMMU_SINGLE; + + if (PageTable != gcvNULL) + { +#if gcdUSE_MMU_EXCEPTION + /* Enable exception */ + _WritePageEntry(PageTable, (1 << 1)); +#else + _WritePageEntry(PageTable, 0); +#endif + } + } + else + { + /* Mark the node as free. */ + node[0] = (pageCount << 8) | gcvMMU_FREE; + node[1] = ~0U; + + if (PageTable != gcvNULL) + { +#if gcdUSE_MMU_EXCEPTION + /* Enable exception */ + gcmkVERIFY_OK(_FillPageTable(PageTable, (gctUINT32)PageCount, 1 << 1)); +#else + gcmkVERIFY_OK(_FillPageTable(PageTable, (gctUINT32)PageCount, 0)); +#endif + } + } + + gcmkDUMP(Mmu->os, + "#[mmu-stlb: free 0x%08X - 0x%08X]", + Address, Address + pageCount * pageSize - 1); + + gcmkDUMP(Mmu->os, + "@[physical.fill 0x%010llX 0x%08X 0x%08X]", + (unsigned long long)(area->stlbPhysical + index * 4), + *(gctUINT32_PTR)PageTable, + pageCount * 4); + + /* We have free nodes. */ + area->freeNodes = gcvTRUE; + + /* Record freed address range. */ + data.addressData.start = Address; + data.addressData.end = Address + (gctUINT32)PageCount * pageSize; + gckQUEUE_Enqueue(&Mmu->recentFreedAddresses, &data); + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); + acquired = gcvFALSE; + +#if gcdENABLE_TRUST_APPLICATION + if (Mmu->hardware->options.secureMode == gcvSECURE_IN_TA) + { + gckKERNEL_SecurityUnmapMemory(Mmu->hardware->kernel, Address, (gctUINT32)PageCount); + } +#endif + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckMMU_AllocatePages( + IN gckMMU Mmu, + IN gctSIZE_T PageCount, + IN gcePAGE_TYPE PageType, + OUT gctPOINTER * PageTable, + OUT gctUINT32 * Address + ) +{ + return gckMMU_AllocatePagesEx( + Mmu, PageCount, gcvVIDMEM_TYPE_GENERIC, PageType, gcvFALSE, PageTable, Address); +} + +gceSTATUS +gckMMU_AllocatePagesEx( + IN gckMMU Mmu, + IN gctSIZE_T PageCount, + IN gceVIDMEM_TYPE Type, + IN gcePAGE_TYPE PageType, + IN gctBOOL Secure, + OUT gctPOINTER * PageTable, + OUT gctUINT32 * Address + ) +{ +#if gcdDISABLE_GPU_VIRTUAL_ADDRESS + gcmkPRINT("GPU virtual address is disabled."); + return gcvSTATUS_NOT_SUPPORTED; +#else + return _AllocatePages(Mmu, PageCount, Type, PageType, Secure, PageTable, Address); +#endif +} + +gceSTATUS +gckMMU_FreePages( + IN gckMMU Mmu, + IN gctBOOL Secure, + IN gcePAGE_TYPE PageType, + IN gctUINT32 Address, + IN gctPOINTER PageTable, + IN gctSIZE_T PageCount + ) +{ + return _FreePages(Mmu, Secure, PageType, Address, PageTable, PageCount); +} + +gceSTATUS +gckMMU_SetPage( + IN gckMMU Mmu, + IN gctPHYS_ADDR_T PageAddress, + IN gcePAGE_TYPE PageType, + IN gctBOOL Writable, + IN gctUINT32 *PageEntry + ) +{ + gctUINT32 addressExt; + gctUINT32 address; + + gcmkHEADER_ARG("Mmu=0x%x", Mmu); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); + gcmkVERIFY_ARGUMENT(PageEntry != gcvNULL); + + if (PageType == gcvPAGE_TYPE_1M) + { + gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFFFF)); + } + else + { + gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFF)); + } + + /* [31:0]. */ + address = (gctUINT32)(PageAddress & 0xFFFFFFFF); + /* [39:32]. */ + addressExt = (gctUINT32)((PageAddress >> 32) & 0xFF); + + if (Mmu->hardware->mmuVersion == 0) + { + _WritePageEntry(PageEntry, address); + } + else + { + _WritePageEntry(PageEntry, _SetPage(address, addressExt, gcvTRUE)); + } + +#ifdef DUMP_IN_KERNEL + { + gctUINT32 *stlbLogical = (PageType == gcvPAGE_TYPE_1M) + ? Mmu->dynamicArea1M.stlbLogical + : Mmu->dynamicArea4K.stlbLogical; + + gctPHYS_ADDR_T stlbPhysical = (PageType == gcvPAGE_TYPE_1M) + ? Mmu->dynamicArea1M.stlbPhysical + : Mmu->dynamicArea4K.stlbPhysical; + + gctPHYS_ADDR_T physical = stlbPhysical + (stlbLogical - PageEntry) * 4; + + gckDUMP(Mmu->os, "@[physical.fill 0x%010llX 0x%08X 4", + physical, *PageEntry); + } +#endif + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckMMU_Flush( + IN gckMMU Mmu, + IN gceVIDMEM_TYPE Type + ) +{ + gckHARDWARE hardware; + gctUINT32 mask; + gctINT i; + + if (Type == gcvVIDMEM_TYPE_VERTEX_BUFFER || + Type == gcvVIDMEM_TYPE_INDEX_BUFFER || + Type == gcvVIDMEM_TYPE_COMMAND) + { + mask = gcvPAGE_TABLE_DIRTY_BIT_FE; + } + else + { + mask = gcvPAGE_TABLE_DIRTY_BIT_OTHER; + } + +#if gcdSHARED_PAGETABLE + for (i = 0; i < gcdMAX_GPU_COUNT; i++) + { + gctUINT j; + hardware = sharedPageTable->hardwares[i]; + if (hardware) + { + for (j = 0; j < gcvENGINE_GPU_ENGINE_COUNT; j++) + { + gcmkVERIFY_OK(gckOS_AtomSetMask(hardware->pageTableDirty[j], mask)); + } + } + } +#else + hardware = Mmu->hardware; + + for (i = 0 ; i < gcvENGINE_GPU_ENGINE_COUNT; i++) + { + gcmkVERIFY_OK( + gckOS_AtomSetMask(hardware->pageTableDirty[i], mask)); + } + + { + gcsLISTHEAD_PTR hardwareHead; + gcmkLIST_FOR_EACH(hardwareHead, &Mmu->hardwareList) + { + hardware = gcmCONTAINEROF(hardwareHead, struct _gckHARDWARE, mmuHead); + + if (hardware != Mmu->hardware) + { + for (i = 0 ; i < gcvENGINE_GPU_ENGINE_COUNT; i++) + { + gcmkVERIFY_OK( + gckOS_AtomSetMask(hardware->pageTableDirty[i], mask)); + } + } + } + } +#endif + + return gcvSTATUS_OK; +} + +gceSTATUS +gckMMU_DumpPageTableEntry( + IN gckMMU Mmu, + IN gceAREA_TYPE AreaType, + IN gctUINT32 Address + ) +{ + gctUINT32_PTR pageTable; + gctUINT32 index; + gctUINT32 mtlb, stlb; + gcsADDRESS_AREA_PTR area = (AreaType == gcvAREA_TYPE_4K) + ? &Mmu->dynamicArea4K : &Mmu->dynamicArea1M; + + gctUINT32 stlbShift = (AreaType == gcvAREA_TYPE_4K) + ? gcdMMU_STLB_4K_SHIFT : gcdMMU_STLB_1M_SHIFT; + + gctUINT32 stlbMask = (AreaType == gcvAREA_TYPE_4K) + ? gcdMMU_STLB_4K_MASK : gcdMMU_STLB_1M_MASK; + + gctUINT32 stlbEntryNum = (AreaType == gcvAREA_TYPE_4K) + ? gcdMMU_STLB_4K_ENTRY_NUM : gcdMMU_STLB_1M_ENTRY_NUM; + + gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address); + gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); + + gcmkASSERT(Mmu->hardware->mmuVersion > 0); + + mtlb = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; + + if (AreaType != gcvAREA_TYPE_FLATMAP) + { + stlb = (Address & stlbMask) >> stlbShift; + + pageTable = area->stlbLogical; + + index = (mtlb - area->mappingStart) + * stlbEntryNum + + stlb; + + gcmkPRINT(" Page table entry = 0x%08X", _ReadPageEntry(pageTable + index)); + } + else + { + gcsMMU_STLB_CHUNK_PTR stlbChunkObj = Mmu->staticSTLB; + gctUINT32 entry = Mmu->mtlbLogical[mtlb]; + + stlb = (Address & gcdMMU_STLB_1M_MASK) >> gcdMMU_STLB_1M_SHIFT; + + entry &= 0xFFFFFFF0; + + while (stlbChunkObj) + { + gctUINT i; + gctBOOL found = gcvFALSE; + for (i = 0; i < stlbChunkObj->mtlbEntryNum; ++i) + { + gctPHYS_ADDR_T stlbPhysBase = stlbChunkObj->physBase + (i * gcdMMU_STLB_1M_SIZE); + gctUINT32_PTR stlbLogical = + (gctUINT32_PTR)((gctUINT8_PTR)stlbChunkObj->logical + (i * gcdMMU_STLB_1M_SIZE)); + if (entry == stlbPhysBase) + { + gcmkPRINT(" Page table entry = 0x%08X", stlbLogical[stlb]); + found = gcvTRUE; + break; + } + } + if (found) + break; + stlbChunkObj = stlbChunkObj->next; + } + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +void +gckMMU_CheckSaftPage( + IN gckMMU Mmu + ) +{ + gctUINT8_PTR safeLogical = Mmu->safePageLogical; + gctUINT32 offsets[] = { + 0, + 64, + 128, + 256, + 2560, + 4000 + }; + + gctUINT32 i = 0; + + while (i < gcmCOUNTOF(offsets)) + { + if (safeLogical[offsets[i]] != 0) + { + gcmkPRINT("%s(%d) safe page is over written [%d] = %x", + __FUNCTION__, __LINE__, i, safeLogical[offsets[i]]); + } + } +} + +void +gckMMU_DumpAddressSpace( + IN gckMMU Mmu + ) +{ + gctUINT i; + gctUINT next; + /* TODO: */ + gcsADDRESS_AREA_PTR area = &Mmu->dynamicArea4K; + gctUINT32_PTR map = area->mapLogical; + gctBOOL used = gcvFALSE; + gctUINT32 numPages; + + /* Grab the mutex. */ + gcmkVERIFY_OK(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); + + /* Find node which contains index. */ + for (i = 0; i < area->stlbEntries; i = next) + { + switch (gcmENTRY_TYPE(map[i])) + { + case gcvMMU_SINGLE: + numPages = 1; + next = i + numPages; + used = gcvFALSE; + break; + + case gcvMMU_FREE: + numPages = map[i] >> 8; + next = i + numPages; + used = gcvFALSE; + break; + + case gcvMMU_USED: + numPages = 1; + next = i + numPages; + used = gcvTRUE; + break; + + default: + gcmkFATAL("MMU table correcupted at index %u!", i); + return; + } + + if (!used) + { + gcmkPRINT("Available Range [%d - %d)", i, i + numPages); + } + } + + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); + +} + +void +gckMMU_DumpRecentFreedAddress( + IN gckMMU Mmu + ) +{ + gckQUEUE queue = &Mmu->recentFreedAddresses; + gctUINT32 i; + gcuQUEUEDATA *data; + + if (queue->count) + { + gcmkPRINT(" Recent %d freed GPU address ranges:", queue->count); + + for (i = 0; i < queue->count; i++) + { + gckQUEUE_GetData(queue, i, &data); + + gcmkPRINT(" [%08X - %08X]", data->addressData.start, data->addressData.end); + } + } +} + +gceSTATUS +gckMMU_FillFlatMapping( + IN gckMMU Mmu, + IN gctUINT64 PhysBase, + IN gctSIZE_T Size + ) +{ + gceSTATUS status; + gckHARDWARE hardware = Mmu->hardware; + + if (hardware->mmuVersion) + { + gcmkONERROR(_FillFlatMapping(Mmu, PhysBase, Size, gcvFALSE, gcvTRUE, gcvNULL)); + } + + return gcvSTATUS_OK; + +OnError: + return status; +} + +gceSTATUS +gckMMU_IsFlatMapped( + IN gckMMU Mmu, + IN gctUINT64 Physical, + IN gctUINT32 Address, + OUT gctBOOL *In + ) +{ + gceSTATUS status; + gctUINT32 i; + gctBOOL inFlatmapping = gcvFALSE; + + gcmkHEADER(); + + gcmkVERIFY_ARGUMENT(In != gcvNULL); + + if (gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_MMU) == gcvFALSE) + { + gcmkONERROR(gcvSTATUS_NOT_SUPPORTED); + } + + if (Physical != gcvINVALID_PHYSICAL_ADDRESS) + { + for (i = 0; i < Mmu->gpuPhysicalRangeCount; i++) + { + if ((Physical >= Mmu->gpuPhysicalRanges[i].start) && + (Physical <= Mmu->gpuPhysicalRanges[i].end)) + { + inFlatmapping = gcvTRUE; + break; + } + } + } + + if (Address != gcvINVALID_ADDRESS) + { + for (i = 0; i < Mmu->gpuAddressRangeCount; i++) + { + if ((Address >= Mmu->gpuAddressRanges[i].start) && + (Address <= Mmu->gpuAddressRanges[i].end)) + { + inFlatmapping = gcvTRUE; + break; + } + } + } + + *In = inFlatmapping; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckMMU_SetupSRAM( + IN gckMMU Mmu, + IN gckHARDWARE Hardware, + IN gckDEVICE Device + ) +{ + gctBOOL needMapInternalSRAM = gcvFALSE; + gctPHYS_ADDR_T reservedBase = gcvINVALID_PHYSICAL_ADDRESS; + gctUINT32 reservedSize = 0; + gctUINT i = 0; + gctUINT j = 0; + gceSTATUS status; + gckKERNEL kernel = Hardware->kernel; + + gcmkHEADER_ARG("Mmu=0x%x Hardware=0x%x", Mmu, Hardware); + + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + + if (Hardware->mmuVersion == 0) + { + gcmkONERROR(gcvSTATUS_OK); + } + + if (!Mmu->sRAMMapped) + { + gctUINT32 address = gcvINVALID_ADDRESS; + gctUINT32 size = 0; + + /* Map all the SRAMs in MMU table. */ + for (i = 0; i < gcvCORE_COUNT; i++) + { + for (j = gcvSRAM_INTERNAL0; j < gcvSRAM_INTER_COUNT; j++) + { + reservedBase = Device->sRAMBases[i][j]; + reservedSize = Device->sRAMSizes[i][j]; + + Device->sRAMBaseAddresses[i][j] = 0; + + needMapInternalSRAM = reservedSize && (reservedBase != gcvINVALID_PHYSICAL_ADDRESS); + + /* Map the internal SRAM. */ + if (needMapInternalSRAM) + { + if (Device->showSRAMMapInfo) + { + gcmkPRINT("Galcore Info: MMU mapped core%d SRAM base=0x%llx size=0x%x", + i, + reservedBase, + reservedSize + ); + } + + /* + * Default gpu virtual base = 0. + * It can be specified if not conflict with existing mapping. + */ + gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical( + Mmu->os, + reservedBase, + &reservedBase + )); + + gcmkONERROR(_FillFlatMapping( + Mmu, + reservedBase, + reservedSize, + gcvTRUE, + gcvTRUE, + &Device->sRAMBaseAddresses[i][j] + )); + + Device->sRAMBases[i][j] = reservedBase; + } + else if (reservedSize && reservedBase == gcvINVALID_PHYSICAL_ADDRESS) + { + /* + * Reserve the internal SRAM range in first reserved MMU mtlb, + * when CPU physical base address is not specified, which means it is reserve usage. + */ + Device->sRAMBaseAddresses[i][j] = (address == gcvINVALID_ADDRESS) ? gcdRESERVE_START : + address + gcmALIGN(size, gcdRESERVE_ALIGN); + + Device->sRAMBases[i][j] = address = Device->sRAMBaseAddresses[i][j]; + + size = Device->sRAMSizes[i][j]; + + Device->sRAMPhysFaked[i][j] = gcvTRUE; + } + +#if gcdCAPTURE_ONLY_MODE + Device->sRAMPhysFaked[i][j] = gcvTRUE; +#endif + } + } + + /* Map all the external SRAMs in MMU table. */ + for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++) + { + if (Device->extSRAMSizes[i] && + (Device->extSRAMBases[i] != gcvINVALID_PHYSICAL_ADDRESS)) + { + + Device->extSRAMBaseAddresses[i] = 0; + + gcmkONERROR(gckOS_CPUPhysicalToGPUPhysical( + Mmu->os, + Device->extSRAMBases[i], + &Device->extSRAMGPUBases[i] + )); + + gcmkONERROR(_FillFlatMapping( + Mmu, + Device->extSRAMGPUBases[i], + Device->extSRAMSizes[i], + gcvFALSE, + gcvTRUE, + &Device->extSRAMBaseAddresses[i] + )); + + Device->extSRAMGPUPhysNames[i] = gckKERNEL_AllocateNameFromPointer(kernel, Device->extSRAMPhysical[i]); + } + } + Mmu->sRAMMapped = gcvTRUE; + } + + /* Get per core SRAM hardware base address. */ + for (i = gcvSRAM_INTERNAL0; i < gcvSRAM_INTER_COUNT; i++) + { + if (Device->sRAMSizes[Hardware->core][i] && + (Device->sRAMBaseAddresses[Hardware->core][i])) + { + kernel->sRAMBaseAddresses[i] = Device->sRAMBaseAddresses[Hardware->core][i]; + kernel->sRAMSizes[i] = Hardware->options.sRAMSizes[i] + = Device->sRAMSizes[Hardware->core][i]; + + kernel->sRAMPhysFaked[i] = Device->sRAMPhysFaked[Hardware->core][i]; + Hardware->options.sRAMGPUVirtAddrs[i] = Device->sRAMBaseAddresses[Hardware->core][i]; + + /* If the internal SRAM usage is reserve. */ + if (kernel->sRAMPhysFaked[i]) + { + /* Use virtual address as the faked physical address which will never be accessed. */ + status = gckVIDMEM_Construct( + Mmu->os, + (gctPHYS_ADDR_T)kernel->sRAMBaseAddresses[i], + kernel->sRAMSizes[i], + 64, + 0, + &kernel->sRAMVidMem[i] + ); + + if (gcmIS_ERROR(status)) + { + kernel->sRAMSizes[i] = 0; + kernel->sRAMVidMem[i] = gcvNULL; + } + else + { + gcmkONERROR(gckOS_RequestReservedMemory( + Mmu->os, + (gctPHYS_ADDR_T)kernel->sRAMBaseAddresses[i], + kernel->sRAMSizes[i], + "Per core SRAM reserve usage region", + gcvTRUE, + &kernel->sRAMPhysical[i] + )); + + kernel->sRAMVidMem[i]->physical = kernel->sRAMPhysical[i]; + } + } + + if (Device->showSRAMMapInfo) + { + gcmkPRINT("Galcore Info: MMU mapped core %d SRAM[%d] hardware virtual address=0x%x size=0x%x", + Hardware->core, + i, + kernel->sRAMBaseAddresses[i], + kernel->sRAMSizes[i] + ); + } + } + } + + for (i = gcvSRAM_EXTERNAL0; i < gcvSRAM_EXT_COUNT; i++) + { + if (Device->extSRAMSizes[i] && + (Device->extSRAMBases[i] != gcvINVALID_PHYSICAL_ADDRESS)) + { + kernel->extSRAMBaseAddresses[i] = Device->extSRAMBaseAddresses[i]; + + Hardware->options.extSRAMGPUVirtAddrs[i] = Device->extSRAMBaseAddresses[i]; + Hardware->options.extSRAMSizes[i] = Device->extSRAMSizes[i]; + Hardware->options.extSRAMCPUPhysAddrs[i] = Device->extSRAMBases[i]; + Hardware->options.extSRAMGPUPhysAddrs[i] = Device->extSRAMGPUBases[i]; + Hardware->options.extSRAMGPUPhysNames[i] = Device->extSRAMGPUPhysNames[i]; + + if (Device->showSRAMMapInfo) + { + gcmkPRINT("Galcore Info: MMU mapped external shared SRAM[%d] CPU view base=0x%llx GPU view base=0x%llx GPU virtual address=0x%x size=0x%x", + i, + Device->extSRAMBases[i], + Device->extSRAMGPUBases[i], + kernel->extSRAMBaseAddresses[i], + Device->extSRAMSizes[i] + ); + } + } + } + + gcsLIST_Add(&Hardware->mmuHead, &Mmu->hardwareList); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the error. */ + gcmkFOOTER(); + return status; +} + + +gceSTATUS +gckMMU_GetPageEntry( + IN gckMMU Mmu, + IN gcePAGE_TYPE PageType, + IN gctUINT32 Address, + IN gctUINT32_PTR *PageTable + ) +{ + gctUINT32_PTR pageTable; + gctUINT32 index; + gctUINT32 mtlbOffset, stlbOffset; + gcsADDRESS_AREA_PTR area = _GetProcessArea(Mmu, PageType, gcvFALSE); + + gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address); + gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); + + gcmkASSERT(Mmu->hardware->mmuVersion > 0); + + mtlbOffset = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; + + if (mtlbOffset >= area->mappingStart) + { + gctUINT32 stlbShift = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_SHIFT : gcdMMU_STLB_4K_SHIFT; + gctUINT32 stlbMask = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_MASK : gcdMMU_STLB_4K_MASK; + gctUINT32 stlbEntryNum = (PageType == gcvPAGE_TYPE_1M) ? gcdMMU_STLB_1M_ENTRY_NUM : gcdMMU_STLB_4K_ENTRY_NUM; + + stlbOffset = (Address & stlbMask) >> stlbShift; + + pageTable = area->stlbLogical; + + index = (mtlbOffset - area->mappingStart) * stlbEntryNum + stlbOffset; + + *PageTable = pageTable + index; + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckMMU_GetAreaType( + IN gckMMU Mmu, + IN gctUINT32 GpuAddress, + OUT gceAREA_TYPE *AreaType + ) +{ + gctUINT32 mtlbIndex; + gctBOOL flatMapped; + gceSTATUS status = gcvSTATUS_OK; + gcsADDRESS_AREA_PTR area4K = &Mmu->dynamicArea4K; +#if gcdENABLE_GPU_1M_PAGE + gcsADDRESS_AREA_PTR area1M = &Mmu->dynamicArea1M; +#endif + + mtlbIndex = _MtlbOffset(GpuAddress); + + gcmkONERROR(gckMMU_IsFlatMapped(Mmu, gcvINVALID_PHYSICAL_ADDRESS, GpuAddress, &flatMapped)); + + if (flatMapped) + { + *AreaType = gcvAREA_TYPE_FLATMAP; + } +#if gcdENABLE_GPU_1M_PAGE + else if (mtlbIndex >= area1M->mappingStart && mtlbIndex <= area1M->mappingEnd) + { + *AreaType = gcvAREA_TYPE_1M; + } +#endif + else if (mtlbIndex >= area4K->mappingStart && mtlbIndex <= area4K->mappingEnd) + { + *AreaType = gcvAREA_TYPE_4K; + } + else + { + *AreaType = gcvAREA_TYPE_UNKNOWN; + } + +OnError: + return status; +}
diff --git a/hal/kernel/gc_hal_kernel_power.c b/hal/kernel/gc_hal_kernel_power.c new file mode 100644 index 0000000..46f497f --- /dev/null +++ b/hal/kernel/gc_hal_kernel_power.c
@@ -0,0 +1,381 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" + +#define _GC_OBJ_ZONE gcvZONE_POWER + +/******************************************************************************\ +************************ Dynamic Voltage Frequency Setting ********************* +\******************************************************************************/ +#if gcdDVFS +static gctUINT32 +_GetLoadHistory( + IN gckDVFS Dvfs, + IN gctUINT32 Select, + IN gctUINT32 Index +) +{ + return Dvfs->loads[Index]; +} + +static void +_IncreaseScale( + IN gckDVFS Dvfs, + IN gctUINT32 Load, + OUT gctUINT8 *Scale + ) +{ + if (Dvfs->currentScale < 32) + { + *Scale = Dvfs->currentScale + 8; + } + else + { + *Scale = Dvfs->currentScale + 8; + *Scale = gcmMIN(64, *Scale); + } +} + +static void +_RecordFrequencyHistory( + gckDVFS Dvfs, + gctUINT32 Frequency + ) +{ + gctUINT32 i = 0; + + struct _FrequencyHistory *history = Dvfs->frequencyHistory; + + for (i = 0; i < 16; i++) + { + if (history->frequency == Frequency) + { + break; + } + + if (history->frequency == 0) + { + history->frequency = Frequency; + break; + } + + history++; + } + + if (i < 16) + { + history->count++; + } +} + +static gctUINT32 +_GetFrequencyHistory( + gckDVFS Dvfs, + gctUINT32 Frequency + ) +{ + gctUINT32 i = 0; + + struct _FrequencyHistory * history = Dvfs->frequencyHistory; + + for (i = 0; i < 16; i++) + { + if (history->frequency == Frequency) + { + break; + } + + history++; + } + + if (i < 16) + { + return history->count; + } + + return 0; +} + +static void +_Policy( + IN gckDVFS Dvfs, + IN gctUINT32 Load, + OUT gctUINT8 *Scale + ) +{ + gctUINT8 load[4], nextLoad; + gctUINT8 scale; + + /* Last 4 history. */ + load[0] = (Load & 0xFF); + load[1] = (Load & 0xFF00) >> 8; + load[2] = (Load & 0xFF0000) >> 16; + load[3] = (Load & 0xFF000000) >> 24; + + /* Determine target scale. */ + if (load[0] > 54) + { + _IncreaseScale(Dvfs, Load, &scale); + } + else + { + nextLoad = (load[0] + load[1] + load[2] + load[3])/4; + + scale = Dvfs->currentScale * (nextLoad) / 54; + + scale = gcmMAX(1, scale); + scale = gcmMIN(64, scale); + } + + Dvfs->totalConfig++; + + Dvfs->loads[(load[0]-1)/8]++; + + *Scale = scale; + + + if (Dvfs->totalConfig % 100 == 0) + { + gcmkPRINT("======================================================="); + gcmkPRINT("GPU Load: %-8d %-8d %-8d %-8d %-8d %-8d %-8d %-8d", + 8, 16, 24, 32, 40, 48, 56, 64); + gcmkPRINT(" %-8d %-8d %-8d %-8d %-8d %-8d %-8d %-8d", + _GetLoadHistory(Dvfs,2, 0), + _GetLoadHistory(Dvfs,2, 1), + _GetLoadHistory(Dvfs,2, 2), + _GetLoadHistory(Dvfs,2, 3), + _GetLoadHistory(Dvfs,2, 4), + _GetLoadHistory(Dvfs,2, 5), + _GetLoadHistory(Dvfs,2, 6), + _GetLoadHistory(Dvfs,2, 7) + ); + + gcmkPRINT("Frequency(MHz) %-8d %-8d %-8d %-8d %-8d", + 58, 120, 240, 360, 480); + gcmkPRINT(" %-8d %-8d %-8d %-8d %-8d", + _GetFrequencyHistory(Dvfs, 58), + _GetFrequencyHistory(Dvfs,120), + _GetFrequencyHistory(Dvfs,240), + _GetFrequencyHistory(Dvfs,360), + _GetFrequencyHistory(Dvfs,480) + ); + } +} + +static void +_TimerFunction( + gctPOINTER Data + ) +{ + gceSTATUS status; + gckDVFS dvfs = (gckDVFS) Data; + gckHARDWARE hardware = dvfs->hardware; + gctUINT32 value; + gctUINT32 frequency; + gctUINT8 scale; + gctUINT32 t1, t2, consumed; + + gckOS_GetTicks(&t1); + + gcmkONERROR(gckHARDWARE_QueryLoad(hardware, &value)); + + /* determine target sacle. */ + _Policy(dvfs, value, &scale); + + /* Set frequency and voltage. */ + gcmkONERROR(gckOS_SetGPUFrequency(hardware->os, hardware->core, scale)); + + /* Query real frequency. */ + gcmkONERROR( + gckOS_QueryGPUFrequency(hardware->os, + hardware->core, + &frequency, + &dvfs->currentScale)); + + _RecordFrequencyHistory(dvfs, frequency); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_POWER, + "Current frequency = %d", + frequency); + + /* Set period. */ + gcmkONERROR(gckHARDWARE_SetDVFSPeroid(hardware, frequency)); + +OnError: + /* Determine next querying time. */ + gckOS_GetTicks(&t2); + + consumed = gcmMIN(((long)t2 - (long)t1), 5); + + if (dvfs->stop == gcvFALSE) + { + gcmkVERIFY_OK(gckOS_StartTimer(hardware->os, + dvfs->timer, + dvfs->pollingTime - consumed)); + } + + return; +} + +gceSTATUS +gckDVFS_Construct( + IN gckHARDWARE Hardware, + OUT gckDVFS * Dvfs + ) +{ + gceSTATUS status; + gctPOINTER pointer; + gckDVFS dvfs = gcvNULL; + gckOS os = Hardware->os; + + gcmkHEADER_ARG("Hardware=0x%X", Hardware); + + gcmkVERIFY_OBJECT(Hardware, gcvOBJ_HARDWARE); + gcmkVERIFY_ARGUMENT(Dvfs != gcvNULL); + + /* Allocate a gckDVFS manager. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckDVFS), &pointer)); + + gckOS_ZeroMemory(pointer, gcmSIZEOF(struct _gckDVFS)); + + dvfs = pointer; + + /* Initialization. */ + dvfs->hardware = Hardware; + dvfs->pollingTime = gcdDVFS_POLLING_TIME; + dvfs->os = Hardware->os; + dvfs->currentScale = 64; + + /* Create a polling timer. */ + gcmkONERROR(gckOS_CreateTimer(os, _TimerFunction, pointer, &dvfs->timer)); + + /* Initialize frequency and voltage adjustment helper. */ + gcmkONERROR(gckOS_PrepareGPUFrequency(os, Hardware->core)); + + /* Return result. */ + *Dvfs = dvfs; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (dvfs) + { + if (dvfs->timer) + { + gcmkVERIFY_OK(gckOS_DestroyTimer(os, dvfs->timer)); + } + + gcmkOS_SAFE_FREE(os, dvfs); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckDVFS_Destroy( + IN gckDVFS Dvfs + ) +{ + gcmkHEADER_ARG("Dvfs=0x%X", Dvfs); + gcmkVERIFY_ARGUMENT(Dvfs != gcvNULL); + + /* Deinitialize helper fuunction. */ + gcmkVERIFY_OK(gckOS_FinishGPUFrequency(Dvfs->os, Dvfs->hardware->core)); + + /* DestroyTimer. */ + gcmkVERIFY_OK(gckOS_DestroyTimer(Dvfs->os, Dvfs->timer)); + + gcmkOS_SAFE_FREE(Dvfs->os, Dvfs); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckDVFS_Start( + IN gckDVFS Dvfs + ) +{ + gcmkHEADER_ARG("Dvfs=0x%X", Dvfs); + gcmkVERIFY_ARGUMENT(Dvfs != gcvNULL); + + gckHARDWARE_InitDVFS(Dvfs->hardware); + + Dvfs->stop = gcvFALSE; + + gckOS_StartTimer(Dvfs->os, Dvfs->timer, Dvfs->pollingTime); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckDVFS_Stop( + IN gckDVFS Dvfs + ) +{ + gcmkHEADER_ARG("Dvfs=0x%X", Dvfs); + gcmkVERIFY_ARGUMENT(Dvfs != gcvNULL); + + Dvfs->stop = gcvTRUE; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} +#endif
diff --git a/hal/kernel/gc_hal_kernel_precomp.h b/hal/kernel/gc_hal_kernel_precomp.h new file mode 100644 index 0000000..0ce3495 --- /dev/null +++ b/hal/kernel/gc_hal_kernel_precomp.h
@@ -0,0 +1,63 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_kernel_precomp_h_ +#define __gc_hal_kernel_precomp_h_ + +#include "gc_hal.h" +#include "shared/gc_hal_driver.h" +#include "gc_hal_kernel.h" + +#endif /* __gc_hal_kernel_precomp_h_ */
diff --git a/hal/kernel/gc_hal_kernel_security.c b/hal/kernel/gc_hal_kernel_security.c new file mode 100644 index 0000000..3c7898e --- /dev/null +++ b/hal/kernel/gc_hal_kernel_security.c
@@ -0,0 +1,286 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" + + + + +#define _GC_OBJ_ZONE gcvZONE_KERNEL + +#if gcdSECURITY + +/* +** Open a security service channel. +*/ +gceSTATUS +gckKERNEL_SecurityOpen( + IN gckKERNEL Kernel, + IN gctUINT32 GPU, + OUT gctUINT32 *Channel + ) +{ + gceSTATUS status; + + gcmkONERROR(gckOS_OpenSecurityChannel(Kernel->os, Kernel->core, Channel)); + gcmkONERROR(gckOS_InitSecurityChannel(*Channel)); + + return gcvSTATUS_OK; + +OnError: + return status; +} + +/* +** Close a security service channel +*/ +gceSTATUS +gckKERNEL_SecurityClose( + IN gctUINT32 Channel + ) +{ + return gcvSTATUS_OK; +} + +/* +** Security service interface. +*/ +gceSTATUS +gckKERNEL_SecurityCallService( + IN gctUINT32 Channel, + IN OUT gcsTA_INTERFACE * Interface +) +{ + gceSTATUS status; + gcmkHEADER(); + + gcmkVERIFY_ARGUMENT(Interface != gcvNULL); + + gckOS_CallSecurityService(Channel, Interface); + + status = Interface->result; + + gcmkONERROR(status); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_SecurityStartCommand( + IN gckKERNEL Kernel + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_START_COMMAND; + iface.u.StartCommand.gpu = Kernel->core; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_SecurityAllocateSecurityMemory( + IN gckKERNEL Kernel, + IN gctUINT32 Bytes, + OUT gctUINT32 * Handle + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_ALLOCATE_SECRUE_MEMORY; + iface.u.AllocateSecurityMemory.bytes = Bytes; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + *Handle = iface.u.AllocateSecurityMemory.memory_handle; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_SecurityExecute( + IN gckKERNEL Kernel, + IN gctPOINTER Buffer, + IN gctUINT32 Bytes + ) +{ + gceSTATUS status; +#if defined(LINUX) + gctPHYS_ADDR_T physical; + gctUINT32 address; +#endif + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_EXECUTE; + iface.u.Execute.command_buffer = (gctUINT32 *)Buffer; + iface.u.Execute.gpu = Kernel->core; + iface.u.Execute.command_buffer_length = Bytes; + +#if defined(LINUX) + gcmkONERROR(gckOS_GetPhysicalAddress(Kernel->os, Buffer, &physical)); + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Kernel->os, physical, &physical)); + gcmkSAFECASTPHYSADDRT(address, physical); + + iface.u.Execute.command_buffer = (gctUINT32 *)address; +#endif + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail( + Kernel->hardware, 0, 0 + )); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_SecurityMapMemory( + IN gckKERNEL Kernel, + IN gctUINT32 *PhysicalArray, + IN gctUINT32 PageCount, + OUT gctUINT32 * GPUAddress + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; +#if defined(LINUX) + gctPHYS_ADDR_T physical; + gctUINT32 address; +#endif + + gcmkHEADER(); + + iface.command = KERNEL_MAP_MEMORY; + +#if defined(LINUX) + gcmkONERROR(gckOS_GetPhysicalAddress(Kernel->os, PhysicalArray, &physical)); + gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Kernel->os, physical, &physical)); + gcmkSAFECASTPHYSADDRT(address, physical); + iface.u.MapMemory.physicals = (gctUINT32 *)address; +#endif + + iface.u.MapMemory.pageCount = PageCount; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + *GPUAddress = iface.u.MapMemory.gpuAddress; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_SecurityUnmapMemory( + IN gckKERNEL Kernel, + IN gctUINT32 GPUAddress, + IN gctUINT32 PageCount + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_UNMAP_MEMORY; + + iface.u.UnmapMemory.gpuAddress = GPUAddress; + iface.u.UnmapMemory.pageCount = PageCount; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +#endif
diff --git a/hal/kernel/gc_hal_kernel_security_v1.c b/hal/kernel/gc_hal_kernel_security_v1.c new file mode 100644 index 0000000..38361ea --- /dev/null +++ b/hal/kernel/gc_hal_kernel_security_v1.c
@@ -0,0 +1,320 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" + + + + +#define _GC_OBJ_ZONE gcvZONE_KERNEL + +#if gcdENABLE_TRUST_APPLICATION + +/* +** Open a security service channel. +*/ +gceSTATUS +gckKERNEL_SecurityOpen( + IN gckKERNEL Kernel, + IN gctUINT32 GPU, + OUT gctUINT32 *Channel + ) +{ + gceSTATUS status; + + gcmkONERROR(gckOS_OpenSecurityChannel(Kernel->os, Kernel->core, Channel)); + gcmkONERROR(gckOS_InitSecurityChannel(*Channel)); + + return gcvSTATUS_OK; + +OnError: + return status; +} + +/* +** Close a security service channel +*/ +gceSTATUS +gckKERNEL_SecurityClose( + IN gctUINT32 Channel + ) +{ + return gcvSTATUS_OK; +} + +/* +** Security service interface. +*/ +gceSTATUS +gckKERNEL_SecurityCallService( + IN gctUINT32 Channel, + IN OUT gcsTA_INTERFACE * Interface +) +{ + gceSTATUS status; + gcmkHEADER(); + + gcmkVERIFY_ARGUMENT(Interface != gcvNULL); + + gckOS_CallSecurityService(Channel, Interface); + + status = Interface->result; + + gcmkONERROR(status); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_SecurityStartCommand( + IN gckKERNEL Kernel, + IN gctUINT32 Address, + IN gctUINT32 Bytes + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_START_COMMAND; + iface.u.StartCommand.gpu = Kernel->core; + iface.u.StartCommand.address = Address; + iface.u.StartCommand.bytes = Bytes; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_SecurityAllocateSecurityMemory( + IN gckKERNEL Kernel, + IN gctUINT32 Bytes, + OUT gctUINT32 * Handle + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_ALLOCATE_SECRUE_MEMORY; + iface.u.AllocateSecurityMemory.bytes = Bytes; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + *Handle = iface.u.AllocateSecurityMemory.memory_handle; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_SecurityMapMemory( + IN gckKERNEL Kernel, + IN gctUINT32 *PhysicalArray, + IN gctPHYS_ADDR_T Physical, + IN gctUINT32 PageCount, + OUT gctUINT32 * GPUAddress + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_MAP_MEMORY; + + iface.u.MapMemory.physicals = PhysicalArray; + iface.u.MapMemory.physical = Physical; + iface.u.MapMemory.pageCount = PageCount; + iface.u.MapMemory.gpuAddress = *GPUAddress; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_SecurityDumpMMUException( + IN gckKERNEL Kernel + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_DUMP_MMU_EXCEPTION; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + + + +gceSTATUS +gckKERNEL_SecurityUnmapMemory( + IN gckKERNEL Kernel, + IN gctUINT32 GPUAddress, + IN gctUINT32 PageCount + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_UNMAP_MEMORY; + + iface.u.UnmapMemory.gpuAddress = GPUAddress; + iface.u.UnmapMemory.pageCount = PageCount; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_ReadMMUException( + IN gckKERNEL Kernel, + IN gctUINT32_PTR MMUStatus, + IN gctUINT32_PTR MMUException + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_READ_MMU_EXCEPTION; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + *MMUStatus = iface.u.ReadMMUException.mmuStatus; + *MMUException = iface.u.ReadMMUException.mmuException; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckKERNEL_HandleMMUException( + IN gckKERNEL Kernel, + IN gctUINT32 MMUStatus, + IN gctPHYS_ADDR_T Physical, + IN gctUINT32 GPUAddress + ) +{ + gceSTATUS status; + gcsTA_INTERFACE iface; + + gcmkHEADER(); + + iface.command = KERNEL_HANDLE_MMU_EXCEPTION; + + iface.u.HandleMMUException.mmuStatus = MMUStatus; + iface.u.HandleMMUException.physical = Physical; + iface.u.HandleMMUException.gpuAddress = GPUAddress; + + gcmkONERROR(gckKERNEL_SecurityCallService(Kernel->securityChannel, &iface)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + + + + +#endif
diff --git a/hal/kernel/gc_hal_kernel_video_memory.c b/hal/kernel/gc_hal_kernel_video_memory.c new file mode 100644 index 0000000..d7dd92b --- /dev/null +++ b/hal/kernel/gc_hal_kernel_video_memory.c
@@ -0,0 +1,4686 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" + +#define _GC_OBJ_ZONE gcvZONE_VIDMEM + +/******************************************************************************\ +******************************* Private Functions ****************************** +\******************************************************************************/ + +/******************************************************************************* +** +** _Split +** +** Split a node on the required byte boundary. +** +** INPUT: +** +** gckOS Os +** Pointer to an gckOS object. +** +** gcuVIDMEM_NODE_PTR Node +** Pointer to the node to split. +** +** gctSIZE_T Bytes +** Number of bytes to keep in the node. +** +** OUTPUT: +** +** Nothing. +** +** RETURNS: +** +** gctBOOL +** gcvTRUE if the node was split successfully, or gcvFALSE if there is an +** error. +** +*/ +static gctBOOL +_Split( + IN gckOS Os, + IN gcuVIDMEM_NODE_PTR Node, + IN gctSIZE_T Bytes + ) +{ + gcuVIDMEM_NODE_PTR node; + gctPOINTER pointer = gcvNULL; + + /* Make sure the byte boundary makes sense. */ + if ((Bytes <= 0) || (Bytes > Node->VidMem.bytes)) + { + return gcvFALSE; + } + + /* Allocate a new gcuVIDMEM_NODE object. */ + if (gcmIS_ERROR(gckOS_Allocate(Os, + gcmSIZEOF(gcuVIDMEM_NODE), + &pointer))) + { + /* Error. */ + return gcvFALSE; + } + + node = pointer; + + /* Initialize gcuVIDMEM_NODE structure. */ + node->VidMem.offset = Node->VidMem.offset + Bytes; + node->VidMem.bytes = Node->VidMem.bytes - Bytes; + node->VidMem.alignment = 0; + node->VidMem.locked = 0; + node->VidMem.parent = Node->VidMem.parent; + node->VidMem.pool = Node->VidMem.pool; + node->VidMem.processID = 0; + node->VidMem.logical = gcvNULL; + node->VidMem.kvaddr = gcvNULL; + + /* Insert node behind specified node. */ + node->VidMem.next = Node->VidMem.next; + node->VidMem.prev = Node; + Node->VidMem.next = node->VidMem.next->VidMem.prev = node; + + /* Insert free node behind specified node. */ + node->VidMem.nextFree = Node->VidMem.nextFree; + node->VidMem.prevFree = Node; + Node->VidMem.nextFree = node->VidMem.nextFree->VidMem.prevFree = node; + + /* Adjust size of specified node. */ + Node->VidMem.bytes = Bytes; + + /* Success. */ + return gcvTRUE; +} + +/******************************************************************************* +** +** _Merge +** +** Merge two adjacent nodes together. +** +** INPUT: +** +** gckOS Os +** Pointer to an gckOS object. +** +** gcuVIDMEM_NODE_PTR Node +** Pointer to the first of the two nodes to merge. +** +** OUTPUT: +** +** Nothing. +** +*/ +static gceSTATUS +_Merge( + IN gckOS Os, + IN gcuVIDMEM_NODE_PTR Node + ) +{ + gcuVIDMEM_NODE_PTR node; + gceSTATUS status; + + /* Save pointer to next node. */ + node = Node->VidMem.next; + + /* This is a good time to make sure the heap is not corrupted. */ + if (Node->VidMem.offset + Node->VidMem.bytes != node->VidMem.offset) + { + /* Corrupted heap. */ + gcmkASSERT( + Node->VidMem.offset + Node->VidMem.bytes == node->VidMem.offset); + return gcvSTATUS_HEAP_CORRUPTED; + } + + /* Adjust byte count. */ + Node->VidMem.bytes += node->VidMem.bytes; + + /* Unlink next node from linked list. */ + Node->VidMem.next = node->VidMem.next; + Node->VidMem.nextFree = node->VidMem.nextFree; + + Node->VidMem.next->VidMem.prev = + Node->VidMem.nextFree->VidMem.prevFree = Node; + + /* Free next node. */ + status = gcmkOS_SAFE_FREE(Os, node); + return status; +} + +/******************************************************************************\ +******************************* gckVIDMEM API Code ****************************** +\******************************************************************************/ + +/******************************************************************************* +** +** gckVIDMEM_Construct +** +** Construct a new gckVIDMEM object. +** +** INPUT: +** +** gckOS Os +** Pointer to an gckOS object. +** +** gctPHYS_ADDR_T PhysicalBase +** Base physical address for the video memory heap. +** +** gctSIZE_T Bytes +** Number of bytes in the video memory heap. +** +** gctSIZE_T Threshold +** Minimum number of bytes beyond am allocation before the node is +** split. Can be used as a minimum alignment requirement. +** +** gctSIZE_T BankSize +** Number of bytes per physical memory bank. Used by bank +** optimization. +** +** OUTPUT: +** +** gckVIDMEM * Memory +** Pointer to a variable that will hold the pointer to the gckVIDMEM +** object. +*/ +gceSTATUS +gckVIDMEM_Construct( + IN gckOS Os, + IN gctPHYS_ADDR_T PhysicalBase, + IN gctSIZE_T Bytes, + IN gctSIZE_T Threshold, + IN gctSIZE_T BankSize, + OUT gckVIDMEM * Memory + ) +{ + gckVIDMEM memory = gcvNULL; + gceSTATUS status; + gcuVIDMEM_NODE_PTR node; + gctINT i, banks = 0; + gctPOINTER pointer = gcvNULL; + gctUINT32 heapBytes; + gctUINT32 bankSize; + gctUINT32 base = 0; + + gcmkHEADER_ARG("Os=0x%x PhysicalBase=%12llx Bytes=%lu Threshold=%lu " + "BankSize=%lu", + Os, PhysicalBase, Bytes, Threshold, BankSize); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Os, gcvOBJ_OS); + gcmkVERIFY_ARGUMENT(Bytes > 0); + gcmkVERIFY_ARGUMENT(Memory != gcvNULL); + + gcmkSAFECASTSIZET(heapBytes, Bytes); + gcmkSAFECASTSIZET(bankSize, BankSize); + + /* Allocate the gckVIDMEM object. */ + gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(struct _gckVIDMEM), &pointer)); + gckOS_ZeroMemory(pointer, gcmSIZEOF(struct _gckVIDMEM)); + + memory = pointer; + + /* Initialize the gckVIDMEM object. */ + memory->object.type = gcvOBJ_VIDMEM; + memory->os = Os; + + /* Set video memory heap information. */ + memory->physicalBase = PhysicalBase; + memory->bytes = heapBytes; + memory->freeBytes = heapBytes; + memory->minFreeBytes = heapBytes; + memory->capability = ~0u; + memory->threshold = Threshold; + memory->mutex = gcvNULL; + + /* Walk all possible banks. */ + for (i = 0; i < gcmCOUNTOF(memory->sentinel); ++i) + { + gctUINT32 bytes; + + if (BankSize == 0) + { + /* Use all bytes for the first bank. */ + bytes = heapBytes; + } + else + { + /* Compute number of bytes for this bank. */ + bytes = gcmALIGN(base + 1, bankSize) - base; + + if (bytes > heapBytes) + { + /* Make sure we don't exceed the total number of bytes. */ + bytes = heapBytes; + } + } + + if (bytes == 0) + { + /* Mark heap is not used. */ + memory->sentinel[i].VidMem.next = + memory->sentinel[i].VidMem.prev = + memory->sentinel[i].VidMem.nextFree = + memory->sentinel[i].VidMem.prevFree = gcvNULL; + continue; + } + + /* Allocate one gcuVIDMEM_NODE union. */ + gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer)); + + node = pointer; + + /* Initialize gcuVIDMEM_NODE union. */ + node->VidMem.parent = memory; + + node->VidMem.next = + node->VidMem.prev = + node->VidMem.nextFree = + node->VidMem.prevFree = &memory->sentinel[i]; + + node->VidMem.offset = base; + node->VidMem.bytes = bytes; + node->VidMem.alignment = 0; + node->VidMem.pool = gcvPOOL_UNKNOWN; + + node->VidMem.locked = 0; + + node->VidMem.processID = 0; + node->VidMem.logical = gcvNULL; + + node->VidMem.kvaddr = gcvNULL; + + /* Initialize the linked list of nodes. */ + memory->sentinel[i].VidMem.next = + memory->sentinel[i].VidMem.prev = + memory->sentinel[i].VidMem.nextFree = + memory->sentinel[i].VidMem.prevFree = node; + + /* Mark sentinel. */ + memory->sentinel[i].VidMem.bytes = 0; + + /* Adjust address for next bank. */ + base += bytes; + heapBytes -= bytes; + banks ++; + } + + /* Assign all the bank mappings. */ + memory->mapping[gcvVIDMEM_TYPE_COLOR_BUFFER] = banks - 1; + memory->mapping[gcvVIDMEM_TYPE_BITMAP] = banks - 1; + + if (banks > 1) --banks; + memory->mapping[gcvVIDMEM_TYPE_DEPTH_BUFFER] = banks - 1; + memory->mapping[gcvVIDMEM_TYPE_HZ_BUFFER] = banks - 1; + + if (banks > 1) --banks; + memory->mapping[gcvVIDMEM_TYPE_TEXTURE] = banks - 1; + + if (banks > 1) --banks; + memory->mapping[gcvVIDMEM_TYPE_VERTEX_BUFFER] = banks - 1; + + if (banks > 1) --banks; + memory->mapping[gcvVIDMEM_TYPE_INDEX_BUFFER] = banks - 1; + + if (banks > 1) --banks; + memory->mapping[gcvVIDMEM_TYPE_TILE_STATUS] = banks - 1; + + if (banks > 1) --banks; + memory->mapping[gcvVIDMEM_TYPE_COMMAND] = banks - 1; + + if (banks > 1) --banks; + memory->mapping[gcvVIDMEM_TYPE_GENERIC] = 0; + + memory->mapping[gcvVIDMEM_TYPE_ICACHE] = 0; + memory->mapping[gcvVIDMEM_TYPE_TXDESC] = 0; + memory->mapping[gcvVIDMEM_TYPE_FENCE] = 0; + memory->mapping[gcvVIDMEM_TYPE_TFBHEADER] = 0; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "[GALCORE] INDEX: bank %d", + memory->mapping[gcvVIDMEM_TYPE_INDEX_BUFFER]); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "[GALCORE] VERTEX: bank %d", + memory->mapping[gcvVIDMEM_TYPE_VERTEX_BUFFER]); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "[GALCORE] TEXTURE: bank %d", + memory->mapping[gcvVIDMEM_TYPE_TEXTURE]); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "[GALCORE] RENDER_TARGET: bank %d", + memory->mapping[gcvVIDMEM_TYPE_COLOR_BUFFER]); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "[GALCORE] DEPTH: bank %d", + memory->mapping[gcvVIDMEM_TYPE_DEPTH_BUFFER]); + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "[GALCORE] TILE_STATUS: bank %d", + memory->mapping[gcvVIDMEM_TYPE_TILE_STATUS]); + + /* Allocate the mutex. */ + gcmkONERROR(gckOS_CreateMutex(Os, &memory->mutex)); + + /* Return pointer to the gckVIDMEM object. */ + *Memory = memory; + + /* Success. */ + gcmkFOOTER_ARG("*Memory=0x%x", *Memory); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (memory != gcvNULL) + { + if (memory->mutex != gcvNULL) + { + /* Delete the mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Os, memory->mutex)); + } + + for (i = 0; i < banks; ++i) + { + /* Free the heap. */ + gcmkASSERT(memory->sentinel[i].VidMem.next != gcvNULL); + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory->sentinel[i].VidMem.next)); + } + + /* Free the object. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, memory)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckVIDMEM_Destroy +** +** Destroy an gckVIDMEM object. +** +** INPUT: +** +** gckVIDMEM Memory +** Pointer to an gckVIDMEM object to destroy. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckVIDMEM_Destroy( + IN gckVIDMEM Memory + ) +{ + gcuVIDMEM_NODE_PTR node, next; + gctINT i; + + gcmkHEADER_ARG("Memory=0x%x", Memory); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM); + + /* Walk all sentinels. */ + for (i = 0; i < gcmCOUNTOF(Memory->sentinel); ++i) + { + /* Bail out of the heap is not used. */ + if (Memory->sentinel[i].VidMem.next == gcvNULL) + { + break; + } + + /* Walk all the nodes until we reach the sentinel. */ + for (node = Memory->sentinel[i].VidMem.next; + node->VidMem.bytes != 0; + node = next) + { + /* Save pointer to the next node. */ + next = node->VidMem.next; + + /* Free the node. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, node)); + } + } + + /* Free the mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Memory->os, Memory->mutex)); + + /* Mark the object as unknown. */ + Memory->object.type = gcvOBJ_UNKNOWN; + + /* Free the gckVIDMEM object. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Memory->os, Memory)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +#if gcdENABLE_BANK_ALIGNMENT + +#if !gcdBANK_BIT_START +#error gcdBANK_BIT_START not defined. +#endif + +#if !gcdBANK_BIT_END +#error gcdBANK_BIT_END not defined. +#endif +/******************************************************************************* +** _GetSurfaceBankAlignment +** +** Return the required offset alignment required to the make BaseAddress +** aligned properly. +** +** INPUT: +** +** gckOS Os +** Pointer to gcoOS object. +** +** gceVIDMEM_TYPE Type +** Type of allocation. +** +** gctUINT32 BaseAddress +** Base address of current video memory node. +** +** OUTPUT: +** +** gctUINT32_PTR AlignmentOffset +** Pointer to a variable that will hold the number of bytes to skip in +** the current video memory node in order to make the alignment bank +** aligned. +*/ +static gceSTATUS +_GetSurfaceBankAlignment( + IN gckKERNEL Kernel, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 BaseAddress, + OUT gctUINT32_PTR AlignmentOffset + ) +{ + gctUINT32 bank; + /* To retrieve the bank. */ + static const gctUINT32 bankMask = (0xFFFFFFFF << gcdBANK_BIT_START) + ^ (0xFFFFFFFF << (gcdBANK_BIT_END + 1)); + + /* To retrieve the bank and all the lower bytes. */ + static const gctUINT32 byteMask = ~(0xFFFFFFFF << (gcdBANK_BIT_END + 1)); + + gcmkHEADER_ARG("Type=%d BaseAddress=0x%x ", Type, BaseAddress); + + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(AlignmentOffset != gcvNULL); + + switch (Type) + { + case gcvVIDMEM_TYPE_COLOR_BUFFER: + bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START); + + /* Align to the first bank. */ + *AlignmentOffset = (bank == 0) ? + 0 : + ((1 << (gcdBANK_BIT_END + 1)) + 0) - (BaseAddress & byteMask); + break; + + case gcvVIDMEM_TYPE_DEPTH_BUFFER: + bank = (BaseAddress & bankMask) >> (gcdBANK_BIT_START); + + /* Align to the third bank. */ + *AlignmentOffset = (bank == 2) ? + 0 : + ((1 << (gcdBANK_BIT_END + 1)) + (2 << gcdBANK_BIT_START)) - (BaseAddress & byteMask); + + /* Minimum 256 byte alignment needed for fast_msaa. */ + if ((gcdBANK_CHANNEL_BIT > 7) || + ((gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_FAST_MSAA) != gcvSTATUS_TRUE) && + (gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_SMALL_MSAA) != gcvSTATUS_TRUE))) + { + /* Add a channel offset at the channel bit. */ + *AlignmentOffset += (1 << gcdBANK_CHANNEL_BIT); + } + break; + + default: + /* no alignment needed. */ + *AlignmentOffset = 0; + } + + /* Return the status. */ + gcmkFOOTER_ARG("*AlignmentOffset=%u", *AlignmentOffset); + return gcvSTATUS_OK; +} +#endif + +static gcuVIDMEM_NODE_PTR +_FindNode( + IN gckKERNEL Kernel, + IN gckVIDMEM Memory, + IN gctINT Bank, + IN gctSIZE_T Bytes, + IN gceVIDMEM_TYPE Type, + IN OUT gctUINT32_PTR Alignment + ) +{ + gcuVIDMEM_NODE_PTR node; + gctUINT32 alignment; + +#if gcdENABLE_BANK_ALIGNMENT + gctUINT32 bankAlignment; + gceSTATUS status; +#endif + + if (Memory->sentinel[Bank].VidMem.nextFree == gcvNULL) + { + /* No free nodes left. */ + return gcvNULL; + } + +#if gcdENABLE_BANK_ALIGNMENT + /* Walk all free nodes until we have one that is big enough or we have + ** reached the sentinel. */ + for (node = Memory->sentinel[Bank].VidMem.nextFree; + node->VidMem.bytes != 0; + node = node->VidMem.nextFree) + { + if (node->VidMem.bytes < Bytes) + { + continue; + } + + gcmkONERROR(_GetSurfaceBankAlignment( + Kernel, + Type, + (gctUINT32)(node->VidMem.parent->physicalBase + node->VidMem.offset), + &bankAlignment)); + + bankAlignment = gcmALIGN(bankAlignment, *Alignment); + + /* Compute number of bytes to skip for alignment. */ + alignment = (*Alignment == 0) + ? 0 + : (*Alignment - (node->VidMem.offset % *Alignment)); + + if (alignment == *Alignment) + { + /* Node is already aligned. */ + alignment = 0; + } + + if (node->VidMem.bytes >= Bytes + alignment + bankAlignment) + { + /* This node is big enough. */ + *Alignment = alignment + bankAlignment; + return node; + } + } +#endif + + /* Walk all free nodes until we have one that is big enough or we have + reached the sentinel. */ + for (node = Memory->sentinel[Bank].VidMem.nextFree; + node->VidMem.bytes != 0; + node = node->VidMem.nextFree) + { + gctUINT offset; + + gctINT modulo; + + gcmkSAFECASTSIZET(offset, node->VidMem.offset); + + modulo = gckMATH_ModuloInt(offset, *Alignment); + + /* Compute number of bytes to skip for alignment. */ + alignment = (*Alignment == 0) ? 0 : (*Alignment - modulo); + + if (alignment == *Alignment) + { + /* Node is already aligned. */ + alignment = 0; + } + + if (node->VidMem.bytes >= Bytes + alignment) + { + /* This node is big enough. */ + *Alignment = alignment; + return node; + } + } + +#if gcdENABLE_BANK_ALIGNMENT +OnError: +#endif + /* Not enough memory. */ + return gcvNULL; +} + +/******************************************************************************* +** +** gckVIDMEM_AllocateLinear +** +** Allocate linear memory from the gckVIDMEM object. +** +** INPUT: +** +** gckVIDMEM Memory +** Pointer to an gckVIDMEM object. +** +** gctSIZE_T Bytes +** Number of bytes to allocate. +** +** gctUINT32 Alignment +** Byte alignment for allocation. +** +** gceVIDMEM_TYPE Type +** Type of surface to allocate (use by bank optimization). +** +** gctUINT32 Flag +** Flag of allocatetion. +** +** gctBOOL Specified +** If user must use this pool, it should set Specified to gcvTRUE, +** otherwise allocator may reserve some memory for other usage, such +** as small block size allocation request. +** +** OUTPUT: +** +** gcuVIDMEM_NODE_PTR * Node +** Pointer to a variable that will hold the allocated memory node. +*/ +static gceSTATUS +gckVIDMEM_AllocateLinear( + IN gckKERNEL Kernel, + IN gckVIDMEM Memory, + IN gctSIZE_T Bytes, + IN gctUINT32 Alignment, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN gctBOOL Specified, + OUT gcuVIDMEM_NODE_PTR * Node + ) +{ + gceSTATUS status; + gcuVIDMEM_NODE_PTR node; + gctUINT32 alignment; + gctINT bank, i; + gctBOOL acquired = gcvFALSE; + gctUINT64 mappingInOne = 1; + + gcmkHEADER_ARG("Memory=0x%x Bytes=%lu Alignment=%u Type=%d", + Memory, Bytes, Alignment, Type); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Memory, gcvOBJ_VIDMEM); + gcmkVERIFY_ARGUMENT(Bytes > 0); + gcmkVERIFY_ARGUMENT(Node != gcvNULL); + gcmkVERIFY_ARGUMENT(Type < gcvVIDMEM_TYPE_COUNT); + + /* Acquire the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Memory->os, Memory->mutex, gcvINFINITE)); + + acquired = gcvTRUE; + + if (Bytes > Memory->freeBytes) + { + /* Not enough memory. */ + status = gcvSTATUS_OUT_OF_MEMORY; + goto OnError; + } + +#if gcdSMALL_BLOCK_SIZE + if ((Memory->freeBytes < (Memory->bytes/gcdRATIO_FOR_SMALL_MEMORY)) + && (Bytes >= gcdSMALL_BLOCK_SIZE) + && (Specified == gcvFALSE) + ) + { + /* The left memory is for small memory.*/ + status = gcvSTATUS_OUT_OF_MEMORY; + goto OnError; + } +#endif + + /* Find the default bank for this surface type. */ + gcmkASSERT((gctINT) Type < gcmCOUNTOF(Memory->mapping)); + bank = Memory->mapping[Type]; + alignment = Alignment; + + /* Find a free node in the default bank. */ + node = _FindNode(Kernel, Memory, bank, Bytes, Type, &alignment); + + /* Out of memory? */ + if (node == gcvNULL) + { + /* Walk all lower banks. */ + for (i = bank - 1; i >= 0; --i) + { + /* Find a free node inside the current bank. */ + node = _FindNode(Kernel, Memory, i, Bytes, Type, &alignment); + if (node != gcvNULL) + { + break; + } + } + } + + if (node == gcvNULL) + { + /* Walk all upper banks. */ + for (i = bank + 1; i < gcmCOUNTOF(Memory->sentinel); ++i) + { + if (Memory->sentinel[i].VidMem.nextFree == gcvNULL) + { + /* Abort when we reach unused banks. */ + break; + } + + /* Find a free node inside the current bank. */ + node = _FindNode(Kernel, Memory, i, Bytes, Type, &alignment); + if (node != gcvNULL) + { + break; + } + } + } + + if (node == gcvNULL) + { + /* Out of memory. */ + status = gcvSTATUS_OUT_OF_MEMORY; + goto OnError; + } + + /* Do we have an alignment? */ + if (alignment > 0) + { + /* Split the node so it is aligned. */ + if (_Split(Memory->os, node, alignment)) + { + /* Successful split, move to aligned node. */ + node = node->VidMem.next; + + /* Remove alignment. */ + alignment = 0; + } + } + + /* Do we have enough memory after the allocation to split it? */ + if (node->VidMem.bytes - Bytes > Memory->threshold) + { + /* Adjust the node size. */ + _Split(Memory->os, node, Bytes); + } + + /* Remove the node from the free list. */ + node->VidMem.prevFree->VidMem.nextFree = node->VidMem.nextFree; + node->VidMem.nextFree->VidMem.prevFree = node->VidMem.prevFree; + node->VidMem.nextFree = + node->VidMem.prevFree = gcvNULL; + + /* Fill in the information. */ + node->VidMem.alignment = alignment; + node->VidMem.parent = Memory; + node->VidMem.logical = gcvNULL; + gcmkONERROR(gckOS_GetProcessID(&node->VidMem.processID)); + + /* Adjust the number of free bytes. */ + Memory->freeBytes -= node->VidMem.bytes; + + if (Memory->freeBytes < Memory->minFreeBytes) + { + Memory->minFreeBytes = Memory->freeBytes; + } + + gckOS_QueryOption(Memory->os, "allMapInOne", &mappingInOne); + if (!mappingInOne) + { + gcmkONERROR(gckOS_RequestReservedMemory( + Memory->os, + Memory->physicalBase + node->VidMem.offset, + node->VidMem.bytes, + "gal reserved memory", + gcvTRUE, + &node->VidMem.physical + )); + } + + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex)); + + /* Return the pointer to the node. */ + *Node = node; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Allocated %u bytes @ 0x%x [0x%08X]", + node->VidMem.bytes, node, node->VidMem.offset); + + /* Success. */ + gcmkFOOTER_ARG("*Node=0x%x", *Node); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Memory->os, Memory->mutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckVIDMEM_AllocateVirtual +** +** Construct a new gcuVIDMEM_NODE union for virtual memory. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gctSIZE_T Bytes +** Number of byte to allocate. +** +** OUTPUT: +** +** gcuVIDMEM_NODE_PTR * Node +** Pointer to a variable that receives the gcuVIDMEM_NODE union pointer. +*/ +static gceSTATUS +gckVIDMEM_AllocateVirtual( + IN gckKERNEL Kernel, + IN gctUINT32 Flag, + IN gctSIZE_T Bytes, + OUT gcuVIDMEM_NODE_PTR * Node + ) +{ + gckOS os; + gceSTATUS status; + gcuVIDMEM_NODE_PTR node = gcvNULL; + gctPOINTER pointer = gcvNULL; + gctINT i; + + gcmkHEADER_ARG("Kernel=0x%x Flag=%x Bytes=%lu", Kernel, Flag, Bytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Bytes > 0); + gcmkVERIFY_ARGUMENT(Node != gcvNULL); + + /* Extract the gckOS object pointer. */ + os = Kernel->os; + gcmkVERIFY_OBJECT(os, gcvOBJ_OS); + + /* Allocate an gcuVIDMEM_NODE union. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer)); + + node = pointer; + + /* Initialize gcuVIDMEM_NODE union for virtual memory. */ + node->Virtual.kernel = Kernel; + node->Virtual.contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS; + node->Virtual.logical = gcvNULL; + node->Virtual.kvaddr = gcvNULL; + node->Virtual.bytes = Bytes; + node->Virtual.secure = (Flag & gcvALLOC_FLAG_SECURITY) != 0; + node->Virtual.onFault = (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT) != 0; + + for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) + { + node->Virtual.lockeds[i] = 0; + node->Virtual.pageTables[i] = gcvNULL; + } + + /* Allocate the virtual memory. */ + gcmkONERROR( + gckOS_AllocatePagedMemory(os, + Flag, + &node->Virtual.bytes, + &node->Virtual.gid, + &node->Virtual.physical)); + + /* Calculate required GPU page (4096) count. */ + /* Assume start address is 4096 aligned. */ + node->Virtual.pageCount = (gctSIZE_T)(((gctUINT64)node->Virtual.bytes + (4096 - 1)) >> 12); + + /* Return pointer to the gcuVIDMEM_NODE union. */ + *Node = node; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Created virtual node 0x%x for %u bytes @ 0x%x", + node, Bytes, node->Virtual.physical); + + /* Success. */ + gcmkFOOTER_ARG("*Node=0x%x", *Node); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (node != gcvNULL) + { + /* Free the structure. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_AddToBlockList( + IN gckKERNEL Kernel, + IN gckVIDMEM_BLOCK VidMemBlock + ) +{ + VidMemBlock->next = Kernel->vidMemBlock; + Kernel->vidMemBlock = VidMemBlock; + + return gcvSTATUS_OK; +} + +static gceSTATUS +_RemoveFromBlockList( + IN gckKERNEL Kernel, + IN gckVIDMEM_BLOCK VidMemBlock + ) +{ + gckVIDMEM_BLOCK vidMemBlock; + gckVIDMEM_BLOCK previous = gcvNULL; + gceHARDWARE_TYPE hwType; + + gcmkVERIFY_OK( + gckKERNEL_GetHardwareType(Kernel, + &hwType)); + + for (vidMemBlock = Kernel->vidMemBlock; vidMemBlock != gcvNULL; vidMemBlock = vidMemBlock->next) + { + if (vidMemBlock->addresses[hwType] == VidMemBlock->addresses[hwType]) + { + if (previous) + { + previous->next = vidMemBlock->next; + } + else + { + Kernel->vidMemBlock = vidMemBlock->next; + } + vidMemBlock->next = gcvNULL; + + break; + } + previous = vidMemBlock; + } + + return gcvSTATUS_OK; +} + +static gckVIDMEM_BLOCK +_FindFreeBlock( + IN gckKERNEL Kernel, + IN gctSIZE_T Bytes + ) +{ + gckVIDMEM_BLOCK vidMemBlock; + + for (vidMemBlock = Kernel->vidMemBlock; vidMemBlock != gcvNULL; vidMemBlock = vidMemBlock->next) + { + if (vidMemBlock->freeBytes >= Bytes) + { + /* Found the block */ + break; + } + } + + return vidMemBlock; +} + +static gceSTATUS +_SplitVirtualChunk( + IN gckOS Os, + IN gcuVIDMEM_NODE_PTR Node, + IN gctSIZE_T Bytes + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gcuVIDMEM_NODE_PTR node = gcvNULL; + gctPOINTER pointer; + gctINT i; + + if ((Bytes <= 0) || (Bytes > Node->VirtualChunk.bytes)) + { + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + /* Allocate a new gcuVIDMEM_NODE object. */ + gcmkONERROR(gckOS_Allocate(Os, + gcmSIZEOF(gcuVIDMEM_NODE), + &pointer)); + + node = pointer; + + /* Intialize the new gcuVIDMEM_NODE. */ + node->VirtualChunk.offset = Node->VirtualChunk.offset + Bytes; + node->VirtualChunk.bytes = Node->VirtualChunk.bytes - Bytes; + node->VirtualChunk.parent = Node->VirtualChunk.parent; + node->VirtualChunk.kernel = Node->VirtualChunk.kernel; + node->VirtualChunk.kvaddr = gcvNULL; + + for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) + { + node->VirtualChunk.lockeds[i] = 0; + } + + /* Insert chunk behind specified chunk. */ + node->VirtualChunk.next = Node->VirtualChunk.next; + node->VirtualChunk.prev = Node; + Node->VirtualChunk.next = node->VirtualChunk.next->VirtualChunk.prev = node; + + /* Insert free chunk behind specified chunk. */ + node->VirtualChunk.nextFree = Node->VirtualChunk.nextFree; + node->VirtualChunk.prevFree = Node; + Node->VirtualChunk.nextFree = node->VirtualChunk.nextFree->VirtualChunk.prevFree = node; + + /* Adjust size of specified chunk. */ + Node->VirtualChunk.bytes = Bytes; + +OnError: + return status; +} + +static gceSTATUS +_MergeVirtualChunk( + IN gckOS Os, + IN gcuVIDMEM_NODE_PTR Node + ) +{ + gcuVIDMEM_NODE_PTR node; + gceSTATUS status = gcvSTATUS_OK; + + node = Node->VirtualChunk.next; + + if (Node->VirtualChunk.offset + Node->VirtualChunk.bytes != node->VirtualChunk.offset) + { + /* Corrupted heap. */ + gcmkASSERT( + Node->VirtualChunk.offset + Node->VirtualChunk.bytes == node->VirtualChunk.offset); + + return gcvSTATUS_HEAP_CORRUPTED; + } + + /* Merge. */ + Node->VirtualChunk.bytes += node->VirtualChunk.bytes; + + /* Unlink next node from linked list. */ + Node->VirtualChunk.next = node->VirtualChunk.next; + Node->VirtualChunk.nextFree = node->VirtualChunk.nextFree; + + Node->VirtualChunk.next->VirtualChunk.prev = + Node->VirtualChunk.nextFree->VirtualChunk.prevFree = Node; + + /* Free the next node. */ + status = gcmkOS_SAFE_FREE(Os, node); + return status; +} + +static gctBOOL +_IsVidMemBlockFree( + IN gckVIDMEM_BLOCK VidMemBlock + ) +{ + return (VidMemBlock->freeBytes == VidMemBlock->bytes); +} + +static gcuVIDMEM_NODE_PTR +_FindVirtualChunkNode( + IN gckKERNEL Kernel, + IN gckVIDMEM_BLOCK VidMemBlock, + IN gctSIZE_T Bytes + ) +{ + gcuVIDMEM_NODE_PTR node; + + if (VidMemBlock->node.VirtualChunk.nextFree == gcvNULL) + { + /* No free chunk left. */ + return gcvNULL; + } + + for (node = VidMemBlock->node.VirtualChunk.nextFree; + node->VirtualChunk.bytes != 0; + node = node->VirtualChunk.nextFree) + { + if (node->VirtualChunk.bytes >= Bytes) + { + /* Got it. */ + return node; + } + } + + return gcvNULL; +} + +/******************************************************************************* +** +** _ConvertPhysical +** +** Convert CPU physical to GPU address for video node. +** +** INPUT: +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gcuVIDMEM_NODE_PTR Node +** Pointer to a gcuVIDMEM_NODE union. +** +** gceCORE Core +** Id of current GPU. +** +** gctPHYS_ADDR_T PhysicalAddress +** CPU physical address +** +** OUTPUT: +** gctUINT32 * Address +** A pointer hold the GPU address. +*/ +static gceSTATUS +_ConvertPhysical( + IN gckKERNEL Kernel, + IN gceCORE Core, + IN gcuVIDMEM_NODE_PTR Node, + IN gckVIDMEM_BLOCK VidMemBlock, + IN gctPHYS_ADDR_T PhysicalAddress, + OUT gctUINT32 * Address + ) +{ + gceSTATUS status; + gctUINT64 physical = 0; + + gcmkHEADER_ARG("Node=0x%X", Node); + + if ((Node && !Node->Virtual.contiguous) || + (VidMemBlock && !VidMemBlock->contiguous)) + { + /* non-contiguous, mapping is required. */ + status = gcvSTATUS_NOT_SUPPORTED; + goto OnError; + } + + if ((Node && Node->Virtual.secure) || + (VidMemBlock && VidMemBlock->secure)) + { + /* Secure, mapping is forced. */ + status = gcvSTATUS_NOT_SUPPORTED; + goto OnError; + } + + /* Convert to GPU physical address. */ + gckOS_CPUPhysicalToGPUPhysical(Kernel->os, PhysicalAddress, &physical); + + + if ((physical > gcvMAXUINT32) || + (Node && (physical + Node->Virtual.bytes - 1 > gcvMAXUINT32)) || + (VidMemBlock && (physical + VidMemBlock->bytes - 1 > gcvMAXUINT32))) + { + /* Above 4G (32bit), mapping is required currently. */ + status = gcvSTATUS_NOT_SUPPORTED; + goto OnError; + } + + if (!gckHARDWARE_IsFeatureAvailable(Kernel->hardware, gcvFEATURE_MMU)) + { + if (physical < Kernel->hardware->baseAddress) + { + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + /* Subtract baseAddress to get a GPU address used for programming. */ + physical -= Kernel->hardware->baseAddress; + + /* 2G upper is virtual space, better to move to gckHARDWARE section. */ + if (physical + Node->Virtual.bytes > 0x80000000) + { + /* End is above 2G, ie virtual space. */ + status = gcvSTATUS_NOT_SUPPORTED; + goto OnError; + } + + *Address = (gctUINT32)physical; + + gcmkFOOTER_ARG("*Address=0x%X", *Address); + return gcvSTATUS_OK; + } + else + { + gctBOOL flatMapped; + + gcmkONERROR(gckMMU_IsFlatMapped(Kernel->mmu, physical, gcvINVALID_ADDRESS, &flatMapped)); + + if (!flatMapped) + { + status = gcvSTATUS_NOT_SUPPORTED; + goto OnError; + } + + *Address = (gctUINT32)physical; + + gcmkFOOTER_ARG("*Address=0x%X", *Address); + return gcvSTATUS_OK; + } + +OnError: + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckVIDMEM_MapVidMemBlock( + IN gckKERNEL Kernel, + IN gckVIDMEM_BLOCK VidMemBlock + ) +{ + gceSTATUS status; + gckOS os = Kernel->os; + gctPHYS_ADDR_T physAddr; + gceHARDWARE_TYPE hwType; + + gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p", Kernel, VidMemBlock); + + gcmkVERIFY_OK( + gckKERNEL_GetHardwareType(Kernel, + &hwType)); + + gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); + gcmkASSERT(VidMemBlock->pageCount > 0); + + gcmkONERROR( + gckOS_GetPhysicalFromHandle(os, + VidMemBlock->physical, + 0, + &physAddr)); + + status = _ConvertPhysical(Kernel, + Kernel->core, + gcvNULL, + VidMemBlock, + physAddr, + &VidMemBlock->addresses[hwType]); + if (gcmIS_ERROR(status)) + { + /* Allocate pages inside the MMU. */ + gcmkONERROR( + gckMMU_AllocatePagesEx(Kernel->mmu, + VidMemBlock->pageCount, + VidMemBlock->type, + gcvPAGE_TYPE_1M, + VidMemBlock->secure, + &VidMemBlock->pageTables[hwType], + &VidMemBlock->addresses[hwType])); + + if (VidMemBlock->onFault != gcvTRUE) + { + /* Map the pages. */ + gcmkONERROR( + gckOS_Map1MPages(os, + Kernel->core, + VidMemBlock->physical, + VidMemBlock->pageCount, + VidMemBlock->addresses[hwType], + VidMemBlock->pageTables[hwType], + gcvTRUE, + VidMemBlock->type)); + } + + gcmkONERROR(gckMMU_Flush(Kernel->mmu, VidMemBlock->type)); + + /* Calculate the GPU virtual address. */ + VidMemBlock->addresses[hwType] |= (gctUINT32) (physAddr & ((1 << 20) - 1)); + } + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Mapped video memory block 0x%x to 0x%08X", + VidMemBlock, + VidMemBlock->addresses[hwType]); + + return gcvSTATUS_OK; + +OnError: + if (VidMemBlock->pageTables[hwType] != gcvNULL) + { + /* Free the pages from the MMU. */ + gcmkVERIFY_OK( + gckMMU_FreePages(Kernel->mmu, + VidMemBlock->secure, + gcvPAGE_TYPE_1M, + VidMemBlock->addresses[hwType], + VidMemBlock->pageTables[hwType], + VidMemBlock->pageCount)); + + VidMemBlock->pageTables[hwType] = gcvNULL; + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_UnmapVidMemBlock( + IN gckMMU Mmu, + IN gceHARDWARE_TYPE HwType, + IN gckVIDMEM_BLOCK VidMemBlock + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Mmu=%p VidMemBlock=%p", Mmu, VidMemBlock); + + gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); + + if (VidMemBlock->pageTables[HwType] != gcvNULL) + { + /* Free the pages from the MMU. */ + gcmkONERROR( + gckMMU_FreePages(Mmu, + VidMemBlock->secure, + gcvPAGE_TYPE_1M, + VidMemBlock->addresses[HwType], + VidMemBlock->pageTables[HwType], + VidMemBlock->pageCount)); + + VidMemBlock->pageTables[HwType] = gcvNULL; + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckVIDMEM_BLOCK_Construct( + IN gckKERNEL Kernel, + IN gctSIZE_T BlockSize, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + OUT gckVIDMEM_BLOCK * VidMemBlock + ) +{ + gceSTATUS status; + gckVIDMEM_BLOCK vidMemBlock = gcvNULL; + gcuVIDMEM_NODE_PTR node = gcvNULL; + gckOS os = Kernel->os; + gctPOINTER pointer; + gctINT i; + + gcmkHEADER_ARG("Kernel=0x%x BlockSize=%lu Type=%x Flag=%x", Kernel, BlockSize, Type); + + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(BlockSize > 0); + gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); + + /* Allocate an gckVIDMEM_BLOCK object. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_BLOCK), &pointer)); + + vidMemBlock = pointer; + + /* Initialize the gckVIDMEM_BLOCK object. */ + vidMemBlock->object.type = gcvOBJ_VIDMEM_BLOCK; + vidMemBlock->os = os; + vidMemBlock->bytes = BlockSize; + vidMemBlock->freeBytes = BlockSize; + /* 1M page count. */ + vidMemBlock->pageCount = (gctUINT32)(BlockSize >> 20); + vidMemBlock->type = Type; + vidMemBlock->contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS; + vidMemBlock->secure = (Flag & gcvALLOC_FLAG_SECURITY) != 0; + vidMemBlock->onFault = (Flag & gcvALLOC_FLAG_ALLOC_ON_FAULT) != 0; + vidMemBlock->mutex = gcvNULL; + vidMemBlock->physical = gcvNULL; + + for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) + { + vidMemBlock->pageTables[i] = gcvNULL; + } + + /* Allocate the mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &vidMemBlock->mutex)); + + /* Allocate one gcuVIDMEM_NODE union. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), &pointer)); + + node = pointer; + + if (!vidMemBlock->contiguous) + { + Flag |= gcvALLOC_FLAG_1M_PAGES; + } + + /* Alloc 1M page size aligned memory block. */ + gcmkONERROR( + gckOS_AllocatePagedMemory(os, + Flag, + &BlockSize, + &vidMemBlock->gid, + &vidMemBlock->physical)); + + /* Map current hardware mmu table with 1M pages for this video memory block. */ + gcmkONERROR(gckVIDMEM_MapVidMemBlock(Kernel, vidMemBlock)); + + /* Initialize gcuVIDMEM_NODE union for virtual memory. */ + node->VirtualChunk.kernel = Kernel; + node->VirtualChunk.offset = 0; + node->VirtualChunk.bytes = BlockSize; + node->VirtualChunk.kvaddr = gcvNULL; + node->VirtualChunk.logical = gcvNULL; + node->VirtualChunk.parent = vidMemBlock; + + for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) + { + node->VirtualChunk.lockeds[i] = 0; + } + + /* Initialize the virtual chunk linked-list. */ + node->VirtualChunk.next = + node->VirtualChunk.prev = + node->VirtualChunk.nextFree = + node->VirtualChunk.prevFree = &vidMemBlock->node; + + vidMemBlock->node.VirtualChunk.next = + vidMemBlock->node.VirtualChunk.prev = + vidMemBlock->node.VirtualChunk.nextFree = + vidMemBlock->node.VirtualChunk.prevFree = node; + + vidMemBlock->node.VirtualChunk.bytes = 0; + + *VidMemBlock = vidMemBlock; + + gcmkFOOTER_ARG("*VidMemBlock=0x%x", *VidMemBlock); + + return gcvSTATUS_OK; + +OnError: + if (vidMemBlock != gcvNULL) + { + if (vidMemBlock->mutex) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(os, vidMemBlock->mutex)); + } + + if (vidMemBlock->physical) + { + gcmkVERIFY_OK(gckOS_FreePagedMemory(os, + vidMemBlock->physical, + vidMemBlock->bytes)); + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, vidMemBlock)); + } + + if (node != gcvNULL) + { + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node)); + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckVIDMEM_BLOCK_Destroy( + IN gckKERNEL Kernel, + IN gckVIDMEM_BLOCK VidMemBlock + ) +{ + gckDEVICE device = Kernel->device; + gctINT i; + + gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p", Kernel, VidMemBlock); + + /* Verify the arguments. */ + gcmkVERIFY_ARGUMENT(Kernel != gcvNULL); + gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); + + if (VidMemBlock->physical) + { + gcmkVERIFY_OK(gckOS_FreePagedMemory(Kernel->os, + VidMemBlock->physical, + VidMemBlock->bytes)); + } + + for (i = 0; i < gcvHARDWARE_NUM_TYPES; i++) + { + if (VidMemBlock->pageTables[i]) + { + gcmkVERIFY_OK(_UnmapVidMemBlock(device->mmus[i], i, VidMemBlock)); + } + } + + /* Free the mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, VidMemBlock->mutex)); + + /* Free the virtual chunk. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, VidMemBlock->node.VirtualChunk.next)); + + /* Free the video memory block. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, VidMemBlock)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +static gceSTATUS +_AllocateVirtualChunk( + IN gckKERNEL Kernel, + IN gckVIDMEM_BLOCK VidMemBlock, + IN gceVIDMEM_TYPE Type, + INOUT gctSIZE_T *Bytes, + OUT gcuVIDMEM_NODE_PTR *Node + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctBOOL acquired = gcvFALSE; + gcuVIDMEM_NODE_PTR node; + gctSIZE_T bytes; + + gcmkHEADER_ARG("Kernel=%p VidMemBlock=%p Type=%x Bytes=%zx", + Kernel, VidMemBlock, Type, *Bytes); + + gcmkVERIFY_ARGUMENT(Node != gcvNULL); + gcmkVERIFY_ARGUMENT(VidMemBlock != gcvNULL); + gcmkVERIFY_ARGUMENT(Bytes > 0); + gcmkVERIFY_ARGUMENT(Type < gcvVIDMEM_TYPE_COUNT); + + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, VidMemBlock->mutex, gcvINFINITE)); + + acquired = gcvTRUE; + + bytes = gcmALIGN(*Bytes, 4096); + + if (bytes > VidMemBlock->freeBytes) + { + /* No enough memory. */ + status = gcvSTATUS_OUT_OF_MEMORY; + goto OnError; + } + + node = _FindVirtualChunkNode(Kernel, VidMemBlock, bytes); + if (node == gcvNULL) + { + gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY); + } + + if (node->VirtualChunk.bytes > bytes) + { + /* Split the chunk. */ + _SplitVirtualChunk(Kernel->os, node, bytes); + } + + /* Remove the chunk from the free list. */ + node->VirtualChunk.prevFree->VirtualChunk.nextFree = node->VirtualChunk.nextFree; + node->VirtualChunk.nextFree->VirtualChunk.prevFree = node->VirtualChunk.prevFree; + node->VirtualChunk.nextFree = node->VirtualChunk.prevFree = gcvNULL; + + /* Fill in the information. */ + node->VirtualChunk.parent = VidMemBlock; + + VidMemBlock->freeBytes -= node->VirtualChunk.bytes; + + *Bytes = bytes; + *Node = node; + +OnError: + if (acquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, VidMemBlock->mutex)); + } + + return status; +} + +static gceSTATUS +gckVIDMEM_AllocateVirtualChunk( + IN gckKERNEL Kernel, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN gctSIZE_T Bytes, + OUT gcuVIDMEM_NODE_PTR * Node + ) +{ + gckOS os; + gceSTATUS status; + gcuVIDMEM_NODE_PTR node; + gckVIDMEM_BLOCK vidMemBlock = gcvNULL; + gctSIZE_T blockSize; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Kernel=0x%x Flag=%x Bytes=%lu", Kernel, Flag, Bytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Bytes > 0); + gcmkVERIFY_ARGUMENT(Node != gcvNULL); + + /* Extract the gckOS object pointer. */ + os = Kernel->os; + gcmkVERIFY_OBJECT(os, gcvOBJ_OS); + + /* Acquire the vidMem block mutex */ + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->vidMemBlockMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Find the free vidmem block. */ + vidMemBlock = _FindFreeBlock(Kernel, Bytes); + if (!vidMemBlock) + { + /* Not found, construct new block. */ + blockSize = gcmALIGN(Bytes, gcd1M_PAGE_SIZE); + + gcmkONERROR( + gckVIDMEM_BLOCK_Construct(Kernel, + blockSize, + Type, + Flag, + &vidMemBlock)); + + gcmkONERROR(_AddToBlockList(Kernel, vidMemBlock)); + } + + /* Allocate virtual chunk node in the found block. */ + gcmkONERROR( + _AllocateVirtualChunk(Kernel, + vidMemBlock, + Type, + &Bytes, + &node)); + + /* Return pointer to the gcuVIDMEM_NODE union. */ + *Node = node; + + /* Release the vidMem block mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex)); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Created virtual node 0x%x for %u bytes @ 0x%x", + node, Bytes, node->Virtual.physical); + + /* Success. */ + gcmkFOOTER_ARG("*Node=0x%x", *Node); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the vidMem block mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckVIDMEM_Free +** +** Free an allocated video memory node. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gcuVIDMEM_NODE_PTR Node +** Pointer to a gcuVIDMEM_NODE object. +** +** OUTPUT: +** +** Nothing. +*/ +static gceSTATUS +gckVIDMEM_Free( + IN gckKERNEL Kernel, + IN gcuVIDMEM_NODE_PTR Node + ) +{ + gceSTATUS status; + gckKERNEL kernel = gcvNULL; + gckVIDMEM memory = gcvNULL; + gckVIDMEM_BLOCK vidMemBlock = gcvNULL; + gcuVIDMEM_NODE_PTR node; + gctBOOL mutexAcquired = gcvFALSE; + gctBOOL vbMutexAcquired = gcvFALSE; + gctBOOL vbListMutexAcquired = gcvFALSE; + gctUINT64 mappingInOne = 1; + + gcmkHEADER_ARG("Node=0x%x", Node); + + /* Verify the arguments. */ + if ((Node == gcvNULL) + || (Node->VidMem.parent == gcvNULL) + ) + { + /* Invalid object. */ + gcmkONERROR(gcvSTATUS_INVALID_OBJECT); + } + + vidMemBlock = Node->VirtualChunk.parent; + + /**************************** Video Memory ********************************/ + + if (Node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + /* Extract pointer to gckVIDMEM object owning the node. */ + memory = Node->VidMem.parent; + + /* Acquire the mutex. */ + gcmkONERROR( + gckOS_AcquireMutex(memory->os, memory->mutex, gcvINFINITE)); + + mutexAcquired = gcvTRUE; + + if (Node->VidMem.kvaddr) + { +#if gcdCAPTURE_ONLY_MODE + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Node->VidMem.kvaddr)); +#else + + gcmkONERROR( + gckOS_DestroyKernelMapping(Kernel->os, + Node->VidMem.parent->physical, + Node->VidMem.kvaddr)); +#endif + + Node->VidMem.kvaddr = gcvNULL; + } + +#ifdef __QNXNTO__ + /* Unmap the video memory. */ + if (Node->VidMem.logical != gcvNULL) + { + gckKERNEL_UnmapVideoMemory( + Kernel, + Node->VidMem.pool, + Node->VidMem.physical, + Node->VidMem.logical, + Node->VidMem.processID, + Node->VidMem.bytes + ); + + Node->VidMem.logical = gcvNULL; + } + + /* Reset. */ + Node->VidMem.processID = 0; + + /* Don't try to re-free an already freed node. */ + if ((Node->VidMem.nextFree == gcvNULL) + && (Node->VidMem.prevFree == gcvNULL) + ) +#endif + { + + /* Check if Node is already freed. */ + if (Node->VidMem.nextFree) + { + /* Node is alread freed. */ + gcmkONERROR(gcvSTATUS_INVALID_DATA); + } + + /* Update the number of free bytes. */ + memory->freeBytes += Node->VidMem.bytes; + + /* Find the next free node. */ + for (node = Node->VidMem.next; + node != gcvNULL && node->VidMem.nextFree == gcvNULL; + node = node->VidMem.next) ; + + if (node == gcvNULL) + { + gcmkONERROR(gcvSTATUS_INVALID_DATA); + } + + /* Insert this node in the free list. */ + Node->VidMem.nextFree = node; + Node->VidMem.prevFree = node->VidMem.prevFree; + + Node->VidMem.prevFree->VidMem.nextFree = + node->VidMem.prevFree = Node; + + /* Is the next node a free node and not the sentinel? */ + if ((Node->VidMem.next == Node->VidMem.nextFree) + && (Node->VidMem.next->VidMem.bytes != 0) + ) + { + /* Merge this node with the next node. */ + gcmkONERROR(_Merge(memory->os, node = Node)); + gcmkASSERT(node->VidMem.nextFree != node); + gcmkASSERT(node->VidMem.prevFree != node); + } + + /* Is the previous node a free node and not the sentinel? */ + if ((Node->VidMem.prev == Node->VidMem.prevFree) + && (Node->VidMem.prev->VidMem.bytes != 0) + ) + { + /* Merge this node with the previous node. */ + gcmkONERROR(_Merge(memory->os, node = Node->VidMem.prev)); + gcmkASSERT(node->VidMem.nextFree != node); + gcmkASSERT(node->VidMem.prevFree != node); + } + } + + gckOS_QueryOption(memory->os, "allMapInOne", &mappingInOne); + if (!mappingInOne) + { + gckOS_ReleaseReservedMemory(memory->os, Node->VidMem.physical); + Node->VidMem.physical = gcvNULL; + } + + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(memory->os, memory->mutex)); + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Node 0x%x is freed.", + Node); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + /* Acquire the vidMem block mutex */ + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->vidMemBlockMutex, gcvINFINITE)); + vbListMutexAcquired = gcvTRUE; + + if (vidMemBlock) + { + gckOS os = vidMemBlock->os; + + gcmkONERROR(gckOS_AcquireMutex(os, vidMemBlock->mutex, gcvINFINITE)); + vbMutexAcquired = gcvTRUE; + kernel = Node->VirtualChunk.kernel; + + if (Node->VirtualChunk.kvaddr) + { + gcmkONERROR( + gckOS_DestroyKernelMapping(kernel->os, + vidMemBlock->physical, + Node->VirtualChunk.kvaddr)); + + Node->VirtualChunk.kvaddr = gcvNULL; + } + + /* Handle the free chunk in the linked-list */ + { + /* Check if chunk is in free list. */ + if (Node->VirtualChunk.nextFree) + { + /* Chunk is already freed. */ + gcmkONERROR(gcvSTATUS_INVALID_DATA); + } + + vidMemBlock->freeBytes += Node->VirtualChunk.bytes; + + /* Find the next free chunk. */ + for (node = Node->VirtualChunk.next; + node != gcvNULL && node->VirtualChunk.nextFree == gcvNULL; + node = node->VirtualChunk.next); + + if (node == gcvNULL) + { + gcmkONERROR(gcvSTATUS_INVALID_DATA); + } + + /* Insert this chunk in the free list. */ + Node->VirtualChunk.nextFree = node; + Node->VirtualChunk.prevFree = node->VirtualChunk.prevFree; + + Node->VirtualChunk.prevFree->VirtualChunk.nextFree = + node->VirtualChunk.prevFree = Node; + + /* Is the next chunk a free chunk. */ + if ((Node->VirtualChunk.next == Node->VirtualChunk.nextFree) + && (Node->VirtualChunk.next->VirtualChunk.bytes != 0) + ) + { + /* Merge this chunk with the next chunk. */ + gcmkONERROR(_MergeVirtualChunk(os, node = Node)); + gcmkASSERT(node->VirtualChunk.nextFree != node); + gcmkASSERT(node->VirtualChunk.prevFree != node); + } + + /* Is the previous chunk a free chunk. */ + if ((Node->VirtualChunk.prev == Node->VirtualChunk.prevFree) + && (Node->VirtualChunk.prev->VirtualChunk.bytes != 0) + ) + { + /* Merge this chunk with the previous chunk. */ + gcmkONERROR(_MergeVirtualChunk(os, node = Node->VirtualChunk.prev)); + gcmkASSERT(node->VirtualChunk.nextFree != node); + gcmkASSERT(node->VirtualChunk.prevFree != node); + } + } + + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, vidMemBlock->mutex)); + + /* Only free the vidmem block when all the chunks are freed. */ + if (_IsVidMemBlockFree(vidMemBlock)) + { + gcmkONERROR(_RemoveFromBlockList(kernel, vidMemBlock)); + + gcmkONERROR(gckVIDMEM_BLOCK_Destroy(kernel, vidMemBlock)); + } + } + + /* Release the vidMem block mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->vidMemBlockMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + /*************************** Virtual Memory *******************************/ + + /* Get gckKERNEL object. */ + kernel = Node->Virtual.kernel; + + /* Verify the gckKERNEL object pointer. */ + gcmkVERIFY_OBJECT(kernel, gcvOBJ_KERNEL); + + + if (Node->Virtual.kvaddr) + { + gcmkVERIFY_OK( + gckOS_DestroyKernelMapping(kernel->os, + Node->Virtual.physical, + Node->Virtual.kvaddr)); + } + + /* Free the virtual memory. */ + gcmkVERIFY_OK(gckOS_FreePagedMemory(kernel->os, + Node->Virtual.physical, + Node->Virtual.bytes)); + + /* Delete the gcuVIDMEM_NODE union. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(kernel->os, Node)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (mutexAcquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex( + memory->os, memory->mutex + )); + } + + if (vbMutexAcquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex( + vidMemBlock->os, vidMemBlock->mutex + )); + } + + if (vbListMutexAcquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex( + vidMemBlock->os, Kernel->vidMemBlockMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckVIDMEM_Lock +** +** Lock a video memory node and return its hardware specific address. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gcuVIDMEM_NODE_PTR Node +** Pointer to a gcuVIDMEM_NODE union. +** +** OUTPUT: +** +** gctUINT32 * Address +** Pointer to a variable that will hold the hardware specific address. +** +** gctUINT32 * PhysicalAddress +** Pointer to a variable that will hold the bus address of a contiguous +** video node. +*/ +static gceSTATUS +gckVIDMEM_Lock( + IN gckKERNEL Kernel, + IN gcuVIDMEM_NODE_PTR Node, + OUT gctUINT32 * Address + ) +{ + gckOS os; + + gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node); + + /* Extract the gckOS object pointer. */ + os = Kernel->os; + + /* Increment the lock count. */ + if (Node->VidMem.locked++ == 0) + { + gctUINT32 address; + gctUINT32 offset = (gctUINT32)Node->VidMem.offset; + + switch (Node->VidMem.pool) + { + case gcvPOOL_LOCAL_EXTERNAL: + address = Kernel->externalBaseAddress + offset; + break; + case gcvPOOL_LOCAL_INTERNAL: + address = Kernel->internalBaseAddress + offset; + break; + case gcvPOOL_INTERNAL_SRAM: + address = Kernel->sRAMBaseAddresses[Kernel->sRAMIndex] + offset; + break; + case gcvPOOL_EXTERNAL_SRAM: + address = Kernel->extSRAMBaseAddresses[Kernel->extSRAMIndex] + offset; + break; + default: + gcmkASSERT(Node->VidMem.pool == gcvPOOL_SYSTEM); + /*FALLTHRU*/ + case gcvPOOL_SYSTEM: + address = Kernel->contiguousBaseAddress + offset; + break; + } + + /* Save address. */ + Node->VidMem.address = address; + } + + *Address = Node->VidMem.address; + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Locked node 0x%x (%d) @ 0x%08X", + Node, + Node->VidMem.locked, + *Address); + + gcmkFOOTER_ARG("*Address=0x%08X", *Address); + return gcvSTATUS_OK; +} + +static gceSTATUS +gckVIDMEM_LockVirtual( + IN gckKERNEL Kernel, + IN gcuVIDMEM_NODE_PTR Node, + OUT gctUINT32 * Address + ) +{ + gceSTATUS status; + gctPHYS_ADDR_T physicalAddress; + gctBOOL locked = gcvFALSE; + gckOS os = Kernel->os; + gceHARDWARE_TYPE hwType; + + gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node); + + gcmkVERIFY_OK( + gckKERNEL_GetHardwareType(Kernel, + &hwType)); + + gcmkONERROR( + gckOS_GetPhysicalFromHandle(os, + Node->Virtual.physical, + 0, + &physicalAddress)); + + + /* Increment the lock count. */ + if (Node->Virtual.lockeds[hwType]++ == 0) + { + locked = gcvTRUE; + + status = _ConvertPhysical( + Kernel, + Kernel->core, + Node, + gcvNULL, + physicalAddress, + &Node->Virtual.addresses[hwType] + ); + + if (gcmIS_ERROR(status)) + { + /* Do GPU address mapping. */ +#if gcdSECURITY + gctPHYS_ADDR physicalArrayPhysical; + gctPOINTER physicalArrayLogical; + + gcmkONERROR(gckOS_AllocatePageArray( + os, + Node->Virtual.physical, + Node->Virtual.pageCount, + &physicalArrayLogical, + &physicalArrayPhysical + )); + + gcmkONERROR(gckKERNEL_SecurityMapMemory( + Kernel, + physicalArrayLogical, + Node->Virtual.pageCount, + &Node->Virtual.addresses[hwType] + )); + + gcmkONERROR(gckOS_FreeNonPagedMemory( + os, + physicalArrayPhysical, + physicalArrayLogical, + 1 + )); +#else + { + /* Allocate pages inside the MMU. */ + gcmkONERROR( + gckMMU_AllocatePagesEx(Kernel->mmu, + Node->Virtual.pageCount, + Node->Virtual.type, + gcvPAGE_TYPE_4K, + Node->Virtual.secure, + &Node->Virtual.pageTables[hwType], + &Node->Virtual.addresses[hwType])); + } + + if (Node->Virtual.onFault != gcvTRUE) + { +#if gcdENABLE_TRUST_APPLICATION + if (Kernel->hardware->options.secureMode == gcvSECURE_IN_TA) + { + gcmkONERROR(gckKERNEL_MapInTrustApplicaiton( + Kernel, + Node->Virtual.logical, + Node->Virtual.physical, + Node->Virtual.addresses[hwType], + Node->Virtual.pageCount + )); + } + else +#endif + { + gcmkDUMP(os, "#[mmu: dynamic mapping: address=0x%08X pageCount=%lu]", + Node->Virtual.addresses[hwType], + (unsigned long)Node->Virtual.pageCount); + + /* Map the pages. */ + gcmkONERROR(gckOS_MapPagesEx(os, + Kernel->core, + Node->Virtual.physical, + Node->Virtual.pageCount, + Node->Virtual.addresses[hwType], + Node->Virtual.pageTables[hwType], + gcvTRUE, + Node->Virtual.type)); + } + } + + { + gcmkONERROR(gckMMU_Flush(Kernel->mmu, Node->Virtual.type)); + } +#endif + + /* GPU MMU page size is fixed at 4096 now. */ + Node->Virtual.addresses[hwType] |= (gctUINT32)physicalAddress & (4096 - 1); + } + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Mapped virtual node 0x%x to 0x%08X", + Node, + Node->Virtual.addresses[hwType]); + } + + /* Return hardware address. */ + *Address = Node->Virtual.addresses[hwType]; + + gcmkFOOTER_ARG("*Address=0x%08X", *Address); + return gcvSTATUS_OK; + +OnError: + if (locked) + { + if (Node->Virtual.pageTables[hwType] != gcvNULL) + { + { + /* Free the pages from the MMU. */ + gcmkVERIFY_OK( + gckMMU_FreePages(Kernel->mmu, + Node->Virtual.secure, + gcvPAGE_TYPE_4K, + Node->Virtual.addresses[hwType], + Node->Virtual.pageTables[hwType], + Node->Virtual.pageCount)); + } + + Node->Virtual.pageTables[hwType] = gcvNULL; + } + + Node->Virtual.lockeds[hwType]--; + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckVIDMEM_LockVirtualChunk( + IN gckKERNEL Kernel, + IN gcuVIDMEM_NODE_PTR Node, + OUT gctUINT32 * Address + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gckVIDMEM_BLOCK vidMemBlock = Node->VirtualChunk.parent; + gceHARDWARE_TYPE hwType; + + gcmkHEADER_ARG("Kernel=%p Node=%p", Kernel, Node); + + gcmkVERIFY_OK( + gckKERNEL_GetHardwareType(Kernel, + &hwType)); + + gcmkASSERT(vidMemBlock != gcvNULL); + + /* Increment the lock count. */ + if (Node->VirtualChunk.lockeds[hwType]++ == 0) + { + if (!vidMemBlock->pageTables[hwType]) + { + /* Map current hardware mmu table with 1M pages for this video memory block. */ + gcmkONERROR(gckVIDMEM_MapVidMemBlock(Kernel, vidMemBlock)); + } + + Node->VirtualChunk.addresses[hwType] = vidMemBlock->addresses[hwType] + + (gctUINT32)Node->VirtualChunk.offset; + } + + /* Return hardware address. */ + *Address = Node->VirtualChunk.addresses[hwType]; + + gcmkFOOTER_ARG("*Address=0x%08X", *Address); + + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckVIDMEM_Unlock +** +** Unlock a video memory node. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gcuVIDMEM_NODE_PTR Node +** Pointer to a locked gcuVIDMEM_NODE union. +** +** gctBOOL * Asynchroneous +** Pointer to a variable specifying whether the surface should be +** unlocked asynchroneously or not. +** +** OUTPUT: +** +** gctBOOL * Asynchroneous +** Pointer to a variable receiving the number of bytes used in the +** command buffer specified by 'Commands'. If gcvNULL, there is no +** command buffer. +*/ +static gceSTATUS +gckVIDMEM_Unlock( + IN gckKERNEL Kernel, + IN gcuVIDMEM_NODE_PTR Node, + IN OUT gctBOOL * Asynchroneous + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d", + Node, gcmOPT_VALUE(Asynchroneous)); + + if (Node->VidMem.locked <= 0) + { + /* The surface was not locked. */ + gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED); + } + + if (Asynchroneous != gcvNULL) + { + /* Schedule an event to sync with GPU. */ + *Asynchroneous = gcvTRUE; + } + else + { + /* Decrement the lock count. */ + Node->VidMem.locked--; + } + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Unlocked node %p (%d)", + Node, + Node->VidMem.locked); + + /* Success. */ + gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckVIDMEM_UnlockVirtual( + IN gckKERNEL Kernel, + IN gcuVIDMEM_NODE_PTR Node, + IN OUT gctBOOL * Asynchroneous + ) +{ + gceSTATUS status; + gceHARDWARE_TYPE hwType; + + gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d", + Node, gcmOPT_VALUE(Asynchroneous)); + + gcmkVERIFY_OK( + gckKERNEL_GetHardwareType(Kernel, + &hwType)); + + if (Asynchroneous != gcvNULL) + { + /* Schedule the surface to be unlocked. */ + *Asynchroneous = gcvTRUE; + } + else + { + if (Node->Virtual.lockeds[hwType] == 0) + { + gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED); + } + + /* Decrement lock count. */ + --Node->Virtual.lockeds[hwType]; + + /* See if we can unlock the resources. */ + if (Node->Virtual.lockeds[hwType] == 0) + { + gctUINT32 address; + + /* Adjust address to page aligned for underlying functions. */ + address = Node->Virtual.addresses[hwType] & ~(4096 - 1); + +#if gcdSECURITY + if (Node->Virtual.addresses[hwType] > 0x80000000) + { + gcmkONERROR(gckKERNEL_SecurityUnmapMemory( + Kernel, + address, + Node->Virtual.pageCount + )); + } +#else + /* Free the page table. */ + if (Node->Virtual.pageTables[hwType] != gcvNULL) + { + { + gcmkONERROR( + gckMMU_FreePages(Kernel->mmu, + Node->Virtual.secure, + gcvPAGE_TYPE_4K, + address, + Node->Virtual.pageTables[hwType], + Node->Virtual.pageCount)); + } + + gcmkONERROR(gckOS_UnmapPages( + Kernel->os, + Node->Virtual.pageCount, + address + )); + + /* Mark page table as freed. */ + Node->Virtual.pageTables[hwType] = gcvNULL; + } +#endif + } + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Unmapped virtual node %p from 0x%08X", + Node, Node->Virtual.addresses[hwType]); + } + + /* Success. */ + gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous)); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckVIDMEM_UnlockVirtualChunk( + IN gckKERNEL Kernel, + IN gcuVIDMEM_NODE_PTR Node, + IN OUT gctBOOL * Asynchroneous + ) +{ + gceSTATUS status; + gceHARDWARE_TYPE hwType; + + gcmkHEADER_ARG("Node=0x%x *Asynchroneous=%d", + Node, gcmOPT_VALUE(Asynchroneous)); + + gcmkVERIFY_OK( + gckKERNEL_GetHardwareType(Kernel, + &hwType)); + + if (Asynchroneous != gcvNULL) + { + /* Schedule an event to sync with GPU. */ + *Asynchroneous = gcvTRUE; + } + else + { + if (Node->VirtualChunk.lockeds[hwType] == 0) + { + /* The surface was not locked. */ + gcmkONERROR(gcvSTATUS_MEMORY_UNLOCKED); + } + + /* Unmap and free pages when video memory free. */ + + /* Decrement the lock count. */ + --Node->VirtualChunk.lockeds[hwType]; + } + + gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_VIDMEM, + "Unlocked node %p (%d)", + Node, + Node->VirtualChunk.lockeds[hwType]); + + /* Success. */ + gcmkFOOTER_ARG("*Asynchroneous=%d", gcmOPT_VALUE(Asynchroneous)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} +/******************************************************************************* +** +** gckVIDMEM_HANDLE_Allocate +** +** Allocate a handle for a gckVIDMEM_NODE object. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** gckVIDMEM_NODE Node +** Pointer to a gckVIDMEM_NODE object. +** +** OUTPUT: +** +** gctUINT32 * Handle +** Pointer to a variable receiving a handle represent this +** gckVIDMEM_NODE in userspace. +*/ +gceSTATUS +gckVIDMEM_HANDLE_Allocate( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE Node, + OUT gctUINT32 * Handle + ) +{ + gceSTATUS status; + gctUINT32 processID = 0; + gctPOINTER pointer = gcvNULL; + gctPOINTER handleDatabase = gcvNULL; + gctPOINTER mutex = gcvNULL; + gctUINT32 handle = 0; + gckVIDMEM_HANDLE handleObject = gcvNULL; + gckOS os = Kernel->os; + + gcmkHEADER_ARG("Kernel=0x%X, Node=0x%X", Kernel, Node); + + gcmkVERIFY_OBJECT(os, gcvOBJ_OS); + + /* Allocate a gckVIDMEM_HANDLE object. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_HANDLE), &pointer)); + + gcmkVERIFY_OK(gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsVIDMEM_HANDLE))); + + handleObject = pointer; + + gcmkONERROR(gckOS_AtomConstruct(os, &handleObject->reference)); + + /* Set default reference count to 1. */ + gckOS_AtomSet(os, handleObject->reference, 1); + + gcmkVERIFY_OK(gckOS_GetProcessID(&processID)); + + gcmkONERROR( + gckKERNEL_FindHandleDatbase(Kernel, + processID, + &handleDatabase, + &mutex)); + + /* Allocate a handle for this object. */ + gcmkONERROR( + gckKERNEL_AllocateIntegerId(handleDatabase, handleObject, &handle)); + + handleObject->node = Node; + handleObject->handle = handle; + + *Handle = handle; + + gcmkFOOTER_ARG("*Handle=%d", *Handle); + return gcvSTATUS_OK; + +OnError: + if (handleObject != gcvNULL) + { + if (handleObject->reference != gcvNULL) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(os, handleObject->reference)); + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, handleObject)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_HANDLE_Reference( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 Handle + ) +{ + gceSTATUS status; + gckVIDMEM_HANDLE handleObject = gcvNULL; + gctPOINTER database = gcvNULL; + gctPOINTER mutex = gcvNULL; + gctINT32 oldValue = 0; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Handle=%d PrcoessID=%d", Handle, ProcessID); + + gcmkONERROR( + gckKERNEL_FindHandleDatbase(Kernel, ProcessID, &database, &mutex)); + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Translate handle to gckVIDMEM_HANDLE object. */ + gcmkONERROR( + gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject)); + + /* Increase the reference count. */ + gckOS_AtomIncrement(Kernel->os, handleObject->reference, &oldValue); + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + acquired = gcvFALSE; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_HANDLE_Dereference( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 Handle + ) +{ + gceSTATUS status; + gctPOINTER handleDatabase = gcvNULL; + gctPOINTER mutex = gcvNULL; + gctINT32 oldValue = 0; + gckVIDMEM_HANDLE handleObject = gcvNULL; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Handle=%d PrcoessID=%d", Handle, ProcessID); + + gcmkONERROR( + gckKERNEL_FindHandleDatbase(Kernel, + ProcessID, + &handleDatabase, + &mutex)); + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Translate handle to gckVIDMEM_HANDLE. */ + gcmkONERROR( + gckKERNEL_QueryIntegerId(handleDatabase, Handle, (gctPOINTER *)&handleObject)); + + gckOS_AtomDecrement(Kernel->os, handleObject->reference, &oldValue); + + if (oldValue == 1) + { + /* Remove handle from database if this is the last reference. */ + gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(handleDatabase, Handle)); + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + acquired = gcvFALSE; + + if (oldValue == 1) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, handleObject->reference)); + gcmkOS_SAFE_FREE(Kernel->os, handleObject); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_HANDLE_Lookup( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN gctUINT32 Handle, + OUT gckVIDMEM_NODE * Node + ) +{ + gceSTATUS status; + gckVIDMEM_HANDLE handleObject = gcvNULL; + gckVIDMEM_NODE node = gcvNULL; + gctPOINTER database = gcvNULL; + gctPOINTER mutex = gcvNULL; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Kernel=0x%X ProcessID=%d Handle=%d", + Kernel, ProcessID, Handle); + + gcmkONERROR( + gckKERNEL_FindHandleDatbase(Kernel, ProcessID, &database, &mutex)); + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); + acquired = gcvTRUE; + + gcmkONERROR( + gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject)); + + node = handleObject->node; + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + acquired = gcvFALSE; + + *Node = node; + + gcmkFOOTER_ARG("*Node=%d", *Node); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_HANDLE_Lookup2( + IN gckKERNEL Kernel, + IN gcsDATABASE_PTR Database, + IN gctUINT32 Handle, + OUT gckVIDMEM_NODE * Node + ) +{ + gceSTATUS status; + gckVIDMEM_HANDLE handleObject = gcvNULL; + gckVIDMEM_NODE node = gcvNULL; + gctPOINTER database = gcvNULL; + gctPOINTER mutex = gcvNULL; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Kernel=0x%X Database=%p Handle=%d", + Kernel, Database, Handle); + + database = Database->handleDatabase; + mutex = Database->handleDatabaseMutex; + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); + acquired = gcvTRUE; + + gcmkONERROR( + gckKERNEL_QueryIntegerId(database, Handle, (gctPOINTER *)&handleObject)); + + node = handleObject->node; + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + acquired = gcvFALSE; + + *Node = node; + + gcmkFOOTER_ARG("*Node=%d", *Node); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + } + + gcmkFOOTER(); + return status; +} + + +static gceSTATUS +gckVIDMEM_NODE_Construct( + IN gckKERNEL Kernel, + IN gcuVIDMEM_NODE_PTR VideoNode, + IN gceVIDMEM_TYPE Type, + IN gcePOOL Pool, + OUT gckVIDMEM_NODE * NodeObject + ) +{ + gceSTATUS status; + gckVIDMEM_NODE node = gcvNULL; + gctPOINTER pointer = gcvNULL; + gckOS os = Kernel->os; + gctUINT i; + + /* Construct a node. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsVIDMEM_NODE), &pointer)); + + gcmkVERIFY_OK(gckOS_ZeroMemory(pointer, gcmSIZEOF(gcsVIDMEM_NODE))); + + node = pointer; + + node->metadata.magic = VIV_VIDMEM_METADATA_MAGIC; + node->metadata.ts_fd = -1; +#ifdef gcdANDROID + node->metadata.ts_address = 0; +#endif + + node->node = VideoNode; + node->kernel = Kernel; + node->type = Type; + node->pool = Pool; + + gcmkONERROR(gckOS_AtomConstruct(os, &node->reference)); + + gcmkONERROR(gckOS_CreateMutex(os, &node->mutex)); + + for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) + { + gcmkONERROR(gckOS_CreateSignal(os, gcvFALSE, &node->sync[i].signal)); + } + + /* Reference is 1 by default . */ + gckOS_AtomSet(os, node->reference, 1); + + gcmkVERIFY_OK( + gckOS_AcquireMutex(Kernel->os, + Kernel->db->videoMemListMutex, + gcvINFINITE)); + + /* Add into video memory node list. */ + gcsLIST_Add(&node->link, &Kernel->db->videoMemList); + + gcmkVERIFY_OK( + gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex)); + + *NodeObject = node; + + return gcvSTATUS_OK; + +OnError: + if (node != gcvNULL) + { + if (node->mutex) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(os, node->mutex)); + } + + if (node->reference != gcvNULL) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(os, node->reference)); + } + + for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) + { + if (node->sync[i].signal != gcvNULL) + { + gcmkVERIFY_OK(gckOS_DestroySignal(os, node->sync[i].signal)); + } + } + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node)); + } + + return status; +} + +gceSTATUS +gckVIDMEM_NODE_AllocateLinear( + IN gckKERNEL Kernel, + IN gckVIDMEM VideoMemory, + IN gcePOOL Pool, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN gctUINT32 Alignment, + IN gctBOOL Specified, + IN OUT gctSIZE_T * Bytes, + OUT gckVIDMEM_NODE * NodeObject + ) +{ + gceSTATUS status; + gctSIZE_T bytes = *Bytes; + gcuVIDMEM_NODE_PTR node = gcvNULL; + gckVIDMEM_NODE nodeObject = gcvNULL; + + gcmkHEADER_ARG("Kernel=%p VideoMemory=%p Pool=%d Alignment=%d Type=%d *Bytes=%u", + Kernel, VideoMemory, Pool, Alignment, Type, bytes); + + gcmkONERROR( + gckVIDMEM_AllocateLinear(Kernel, + VideoMemory, + bytes, + Alignment, + Type, + Flag, + Specified, + &node)); + + /* Update pool. */ + node->VidMem.pool = Pool; + bytes = node->VidMem.bytes; + + /* Construct a node. */ + gcmkONERROR( + gckVIDMEM_NODE_Construct(Kernel, node, Type, Pool, &nodeObject)); + + *Bytes = bytes; + *NodeObject = nodeObject; + + gcmkFOOTER_ARG("*Bytes=%u *NodeObject=%p", bytes, nodeObject); + return gcvSTATUS_OK; + +OnError: + if (node) + { + gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_AllocateVirtual( + IN gckKERNEL Kernel, + IN gcePOOL Pool, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN OUT gctSIZE_T * Bytes, + OUT gckVIDMEM_NODE * NodeObject + ) +{ + gceSTATUS status; + gctSIZE_T bytes = *Bytes; + gcuVIDMEM_NODE_PTR node = gcvNULL; + gckVIDMEM_NODE nodeObject = gcvNULL; + + gcmkHEADER_ARG("Kernel=%p Pool=%d Type=%d Flag=%x *Bytes=%u", + Kernel, Pool, Type, Flag, bytes); + + gcmkONERROR( + gckVIDMEM_AllocateVirtual(Kernel, Flag, bytes, &node)); + + /* Update type. */ + node->Virtual.type = Type; + bytes = node->Virtual.bytes; + + /* Construct a node. */ + gcmkONERROR( + gckVIDMEM_NODE_Construct(Kernel, node, Type, Pool, &nodeObject)); + + *Bytes = bytes; + *NodeObject = nodeObject; + + gcmkFOOTER_ARG("*Bytes=%u *NodeObject=%p", bytes, nodeObject); + return gcvSTATUS_OK; + +OnError: + if (node) + { + gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_AllocateVirtualChunk( + IN gckKERNEL Kernel, + IN gcePOOL Pool, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN OUT gctSIZE_T * Bytes, + OUT gckVIDMEM_NODE * NodeObject + ) +{ + gceSTATUS status; + gctSIZE_T bytes = *Bytes; + gcuVIDMEM_NODE_PTR node = gcvNULL; + gckVIDMEM_NODE nodeObject = gcvNULL; + + gcmkHEADER_ARG("Kernel=%p Pool=%d Type=%d Flag=%x *Bytes=%u", + Kernel, Pool, Type, Flag, bytes); + + gcmkONERROR( + gckVIDMEM_AllocateVirtualChunk(Kernel, Type, Flag, bytes, &node)); + + bytes = node->VirtualChunk.bytes; + + /* Construct a node. */ + gcmkONERROR( + gckVIDMEM_NODE_Construct(Kernel, node, Type, Pool, &nodeObject)); + + *Bytes = bytes; + *NodeObject = nodeObject; + + gcmkFOOTER_ARG("*Bytes=%u *NodeObject=%p", bytes, nodeObject); + return gcvSTATUS_OK; + +OnError: + if (node) + { + gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_Reference( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject + ) +{ + gctINT32 oldValue; + gcmkHEADER_ARG("Kernel=0x%X NodeObject=0x%X", Kernel, NodeObject); + + gcmkVERIFY_ARGUMENT(NodeObject != gcvNULL); + + gckOS_AtomIncrement(Kernel->os, NodeObject->reference, &oldValue); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckVIDMEM_NODE_Dereference( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject + ) +{ + gctINT32 oldValue = 0; + gctPOINTER database = Kernel->db->nameDatabase; + gctPOINTER mutex = Kernel->db->nameDatabaseMutex; + gctUINT i; + + gcmkHEADER_ARG("Kernel=0x%X NodeObject=0x%X", Kernel, NodeObject); + gcmkVERIFY_ARGUMENT(NodeObject != gcvNULL); + + gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); + + gcmkVERIFY_OK(gckOS_AtomDecrement(Kernel->os, NodeObject->reference, &oldValue)); + + if (oldValue == 1 && NodeObject->name) + { + /* Free name if exists. */ + gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(database, NodeObject->name)); + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + + if (oldValue == 1) + { + gcmkVERIFY_OK( + gckOS_AcquireMutex(Kernel->os, + Kernel->db->videoMemListMutex, + gcvINFINITE)); + + /* Remove from video memory node list. */ + gcsLIST_Del(&NodeObject->link); + + gcmkVERIFY_OK( + gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex)); + + /* Free gcuVIDMEM_NODE. */ + gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, NodeObject->node)); + + gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, NodeObject->reference)); + + gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, NodeObject->mutex)); + + for (i = 0; i < gcvENGINE_GPU_ENGINE_COUNT; i++) + { + if (NodeObject->sync[i].signal != gcvNULL) + { + gcmkVERIFY_OK(gckOS_DestroySignal(Kernel->os, NodeObject->sync[i].signal)); + } + } + + /* Should not cause recursive call since tsNode->tsNode should be NULL */ + if (NodeObject->tsNode) + { + gcmkASSERT(!NodeObject->tsNode->tsNode); + gckVIDMEM_NODE_Dereference(Kernel, NodeObject->tsNode); + } + + gcmkOS_SAFE_FREE(Kernel->os, NodeObject); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +gceSTATUS +gckVIDMEM_NODE_GetReference( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctINT32 * ReferenceCount + ) +{ + gctINT32 value; + + gckOS_AtomGet(Kernel->os, NodeObject->reference, &value); + + *ReferenceCount = value; + return gcvSTATUS_OK; +} + +gceSTATUS +gckVIDMEM_NODE_Lock( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctUINT32 * Address + ) +{ + gceSTATUS status; + gckOS os = Kernel->os; + gctBOOL acquired = gcvFALSE; + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + + gcmkHEADER_ARG("NodeObject=%p", NodeObject); + + /* Grab the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); + acquired = gcvTRUE; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + gcmkONERROR(gckVIDMEM_Lock(Kernel, node, Address)); + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + gcmkONERROR(gckVIDMEM_LockVirtualChunk(Kernel, node, Address)); + } + else + { + gcmkONERROR(gckVIDMEM_LockVirtual(Kernel, node, Address)); + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + + gcmkFOOTER_ARG("*Address=0x%08X", *Address); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_Unlock( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctUINT32 ProcessID, + IN OUT gctBOOL * Asynchroneous + ) +{ + gceSTATUS status; + gckOS os = Kernel->os; + gctBOOL acquired = gcvFALSE; + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + + gcmkHEADER_ARG("NodeObject=%p Asynchroneous=%p", NodeObject, Asynchroneous); + + /* Grab the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); + acquired = gcvTRUE; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + gcmkONERROR(gckVIDMEM_Unlock(Kernel, node, Asynchroneous)); + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + gcmkONERROR(gckVIDMEM_UnlockVirtualChunk(Kernel, node, Asynchroneous)); + } + else + { + gcmkONERROR(gckVIDMEM_UnlockVirtual(Kernel, node, Asynchroneous)); + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + + gcmkFOOTER_ARG("*Asynchroneous=0x%08X", gcmOPT_VALUE(Asynchroneous)); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_CleanCache( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctSIZE_T Offset, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ) +{ + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gctPHYS_ADDR physHandle = gcvNULL; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + gceSTATUS status; + + gcmkHEADER_ARG("Kernel=%p NodeObject=%d Offset=0x%llx Logical=%p Bytes=0x%llx", + Kernel, NodeObject, Offset, Logical, Bytes); + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + gcmkONERROR(gckOS_MemoryBarrier(Kernel->os, Logical)); + + /* Reserved pool can't be cacheable */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + physHandle = vidMemBlock->physical; + } + else + { + physHandle = node->Virtual.physical; + } + + gcmkONERROR(gckOS_CacheFlush( + Kernel->os, + 0, + physHandle, + Offset, + Logical, + Bytes + )); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_InvalidateCache( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctSIZE_T Offset, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ) +{ + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gctPHYS_ADDR physHandle = gcvNULL; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + gceSTATUS status; + + gcmkHEADER_ARG("Kernel=%p NodeObject=%d Offset=0x%llx Logical=%p Bytes=0x%llx", + Kernel, NodeObject, Offset, Logical, Bytes); + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + /* Reserved pool can't be cacheable */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + physHandle = vidMemBlock->physical; + } + else + { + physHandle = node->Virtual.physical; + } + + gcmkONERROR(gckOS_CacheInvalidate( + Kernel->os, + 0, + physHandle, + Offset, + Logical, + Bytes + )); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_GetLockCount( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctINT32 * LockCount + ) +{ + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + gctINT32 lockCount = 0; + gctINT i = 0; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + lockCount = node->VidMem.locked; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + for (; i < gcvHARDWARE_NUM_TYPES; i++) + { + lockCount += node->VirtualChunk.lockeds[i]; + } + } + else + { + + for (; i < gcvHARDWARE_NUM_TYPES; i++) + { + lockCount += node->Virtual.lockeds[i]; + } + } + + *LockCount = lockCount; + + return gcvSTATUS_OK; +} + +gceSTATUS +gckVIDMEM_NODE_LockCPU( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctBOOL Cacheable, + IN gctBOOL FromUser, + OUT gctPOINTER * Logical + ) +{ + gceSTATUS status; + gckOS os = Kernel->os; + gctBOOL acquired = gcvFALSE; + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + gctPOINTER logical = gcvNULL; + + gcmkHEADER_ARG("NodeObject=%p", NodeObject); + + /* Grab the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); + acquired = gcvTRUE; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + if (Cacheable == gcvTRUE) + { + gcmkONERROR(gcvSTATUS_INVALID_REQUEST); + } + + if (FromUser) + { +#if gcdCAPTURE_ONLY_MODE + node->VidMem.logical = NodeObject->captureLogical; +#else + /* Map video memory pool to user space. */ + gcmkONERROR( + gckKERNEL_MapVideoMemory(Kernel, + gcvTRUE, + node->VidMem.pool, + node->VidMem.physical, + (gctUINT32)node->VidMem.offset, + (gctUINT32)node->VidMem.bytes, + &node->VidMem.logical)); +#endif + + logical = node->VidMem.logical; + } + else + { + /* Map video memory pool to kernel space. */ + if (!node->VidMem.kvaddr) + { +#if gcdCAPTURE_ONLY_MODE + gcmkONERROR(gckOS_Allocate(os, + node->VidMem.bytes, + &node->VidMem.kvaddr)); +#else + gcmkONERROR( + gckOS_CreateKernelMapping(os, + node->VidMem.parent->physical, + node->VidMem.offset, + node->VidMem.bytes, + &node->VidMem.kvaddr)); +#endif + } + + logical = node->VidMem.kvaddr; + } + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + if (FromUser) + { + /* Lock the entire vidmem block. */ + gcmkONERROR( + gckOS_LockPages(os, + vidMemBlock->physical, + vidMemBlock->bytes, + Cacheable, + &logical)); + + /* Get the logical with offset in block. */ + logical = (uint8_t *)logical + node->VirtualChunk.offset; + node->VirtualChunk.logical = logical; + } + else + { + /* Map once and will cancel map when free. */ + if (!node->VirtualChunk.kvaddr) + { + gcmkONERROR( + gckOS_CreateKernelMapping(os, + vidMemBlock->physical, + node->VirtualChunk.offset, + node->VirtualChunk.bytes, + &node->VirtualChunk.kvaddr)); + } + + logical = node->VirtualChunk.kvaddr; + } + } + else + { + if (FromUser) + { + gcmkONERROR( + gckOS_LockPages(os, + node->Virtual.physical, + node->Virtual.bytes, + Cacheable, + &logical)); + + node->Virtual.logical = logical; + } + else + { + /* Map once and will cancel map when free. */ + if (!node->Virtual.kvaddr) + { + gcmkONERROR( + gckOS_CreateKernelMapping(os, + node->Virtual.physical, + 0, + node->Virtual.bytes, + &node->Virtual.kvaddr)); + } + + logical = node->Virtual.kvaddr; + } + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + + *Logical = logical; + + gcmkFOOTER_ARG("*Logical=%p", logical); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_UnlockCPU( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctUINT32 ProcessID, + IN gctBOOL FromUser, + IN gctBOOL Defer + ) +{ + gceSTATUS status; + gckOS os = Kernel->os; + gctBOOL acquired = gcvFALSE; + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + + gcmkHEADER_ARG("NodeObject=%p", NodeObject); + + /* Grab the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); + acquired = gcvTRUE; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + if (FromUser) + { +#if gcdCAPTURE_ONLY_MODE || defined __QNXNTO__ + /* Do nothing here. */ +#else + if (!Defer) + { + /* Unmap the video memory. */ + if (node->VidMem.logical != gcvNULL) + { + gckKERNEL_UnmapVideoMemory( + Kernel, + node->VidMem.pool, + node->VidMem.physical, + node->VidMem.logical, + node->VidMem.processID, + node->VidMem.bytes + ); + + node->VidMem.logical = gcvNULL; + } + /* Reset. */ + node->VidMem.processID = 0; + } +#endif + } + else + { + /* + * Kernel side may lock for CPU access for multiple times. Since + * we don't have lock counts currently, we don't cancel CPU + * mapping here, and will cancel at 'free' instead. + */ + /* + gcmkONERROR( + gckOS_DestroyKernelMapping(os, + node->VidMem.parent->physical, + node->VidMem.kvaddr)); + + node->VidMem.kvaddr = gcvNULL; + */ + } + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + if (FromUser) + { + gcmkONERROR( + gckOS_UnlockPages(os, + vidMemBlock->physical, + vidMemBlock->bytes, + node->VirtualChunk.logical)); + } + else + { + /* Nothing to do. */ + } + } + else + { + if (FromUser) + { + gcmkONERROR( + gckOS_UnlockPages(os, + node->Virtual.physical, + node->Virtual.bytes, + node->Virtual.logical)); + } + else + { + /* + * Kernel side may lock for CPU access for multiple times. Since + * we don't have lock counts currently, we don't cancel CPU + * mapping here, and will cancel at 'free' instead. + */ + /* + gcmkONERROR( + gckOS_DestroyKernelMapping(os, + node->Virtual.physical, + node->Virtual.kvaddr)); + + node->Virtual.kvaddr = gcvNULL; + */ + } + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_GetPhysical( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctUINT32 Offset, + OUT gctPHYS_ADDR_T * PhysicalAddress + ) +{ + gceSTATUS status; + gckOS os = Kernel->os; + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + + gcmkHEADER_ARG("NodeObject=%p", NodeObject); + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + if (Offset >= node->VidMem.bytes) + { + /* Exceeds node size. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + *PhysicalAddress = node->VidMem.parent->physicalBase + + node->VidMem.offset + + Offset; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + if (Offset >= node->VirtualChunk.bytes) + { + /* Exceeds node size. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + gcmkONERROR( + gckOS_GetPhysicalFromHandle(os, + vidMemBlock->physical, + (gctUINT32)node->VirtualChunk.offset + Offset, + PhysicalAddress)); + } + else + { + if (Offset >= node->Virtual.bytes) + { + /* Exceeds node size. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + gcmkONERROR( + gckOS_GetPhysicalFromHandle(os, + node->Virtual.physical, + Offset, + PhysicalAddress)); + } + + gcmkFOOTER_ARG("*PhysicalAddress=0x%llx", *PhysicalAddress); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_GetGid( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctUINT32 * Gid + ) +{ + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + *Gid = 0; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + *Gid = vidMemBlock->gid; + } + else + { + *Gid = node->Virtual.gid; + } + + return gcvSTATUS_OK; +} + +gceSTATUS +gckVIDMEM_NODE_GetSize( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctSIZE_T * Size + ) +{ + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + *Size = node->VidMem.bytes; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + *Size = node->VirtualChunk.bytes; + } + else + { + *Size = node->Virtual.bytes; + } + + return gcvSTATUS_OK; +} + +gceSTATUS +gckVIDMEM_NODE_GetType( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gceVIDMEM_TYPE * Type, + OUT gcePOOL * Pool + ) +{ + if (Type) + { + *Type = NodeObject->type; + } + + if (Pool) + { + *Pool = NodeObject->pool; + } + + return gcvSTATUS_OK; +} + +#if defined(CONFIG_DMA_SHARED_BUFFER) + +/******************************************************************************* +** +** +** Code for dma_buf ops +** +** +*******************************************************************************/ + +#include <linux/slab.h> +#include <linux/mm_types.h> +#include <linux/dma-buf.h> + +static struct sg_table *_dmabuf_map(struct dma_buf_attachment *attachment, + enum dma_data_direction direction) +{ + struct sg_table *sgt = gcvNULL; + struct dma_buf *dmabuf = attachment->dmabuf; + gckVIDMEM_NODE nodeObject = dmabuf->priv; + gceSTATUS status = gcvSTATUS_OK; + + do + { + gcuVIDMEM_NODE_PTR node = nodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + gctPHYS_ADDR physical = gcvNULL; + gctSIZE_T offset = 0; + gctSIZE_T bytes = 0; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + physical = node->VidMem.parent->physical; + offset = node->VidMem.offset; + bytes = node->VidMem.bytes; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + physical = vidMemBlock->physical; + offset = node->VirtualChunk.offset; + bytes = node->VirtualChunk.bytes; + } + else + { + physical = node->Virtual.physical; + offset = 0; + bytes = node->Virtual.bytes; + } + + gcmkERR_BREAK(gckOS_MemoryGetSGT(nodeObject->kernel->os, physical, offset, bytes, (gctPOINTER*)&sgt)); + + if (dma_map_sg(attachment->dev, sgt->sgl, sgt->nents, direction) == 0) + { + sg_free_table(sgt); + kfree(sgt); + sgt = gcvNULL; + gcmkERR_BREAK(gcvSTATUS_GENERIC_IO); + } + } + while (gcvFALSE); + + return sgt; +} + +static void _dmabuf_unmap(struct dma_buf_attachment *attachment, + struct sg_table *sgt, + enum dma_data_direction direction) +{ + dma_unmap_sg(attachment->dev, sgt->sgl, sgt->nents, direction); + + sg_free_table(sgt); + kfree(sgt); +} + +static int _dmabuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + gckVIDMEM_NODE nodeObject = dmabuf->priv; + gcuVIDMEM_NODE_PTR node = nodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + gctPHYS_ADDR physical = gcvNULL; + gctSIZE_T skipPages = vma->vm_pgoff; + gctSIZE_T numPages = PAGE_ALIGN(vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + gceSTATUS status = gcvSTATUS_OK; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + physical = node->VidMem.parent->physical; + skipPages += (node->VidMem.offset >> PAGE_SHIFT); + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + physical = vidMemBlock->physical; + skipPages += (node->VirtualChunk.offset >> PAGE_SHIFT); + } + else + { + physical = node->Virtual.physical; + } + + gcmkONERROR(gckOS_MemoryMmap(nodeObject->kernel->os, physical, skipPages, numPages, vma)); + +OnError: + return gcmIS_ERROR(status) ? -EINVAL : 0; +} + +static void _dmabuf_release(struct dma_buf *dmabuf) +{ + gckVIDMEM_NODE nodeObject = dmabuf->priv; + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(nodeObject->kernel, nodeObject)); +} + +#if LINUX_VERSION_CODE <= KERNEL_VERSION(5,5,7) +static void *_dmabuf_kmap(struct dma_buf *dmabuf, unsigned long offset) +{ + gckVIDMEM_NODE nodeObject = dmabuf->priv; + gcuVIDMEM_NODE_PTR node = nodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + gctINT8_PTR kvaddr = gcvNULL; + gctPHYS_ADDR physical = gcvNULL; + gctSIZE_T bytes = 0; + + offset = (offset << PAGE_SHIFT); + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + physical = node->VidMem.parent->physical; + offset += node->VidMem.offset; + bytes = node->VidMem.bytes; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + physical = vidMemBlock->physical; + offset += node->VirtualChunk.offset; + bytes = node->VirtualChunk.bytes; + } + else + { + physical = node->Virtual.physical; + bytes = node->Virtual.bytes; + } + + if (gcmIS_SUCCESS(gckOS_CreateKernelMapping( + nodeObject->kernel->os, physical, 0, bytes, (gctPOINTER*)&kvaddr))) + { + kvaddr += offset; + } + + return (gctPOINTER)kvaddr; +} + +static void _dmabuf_kunmap(struct dma_buf *dmabuf, unsigned long offset, void *ptr) +{ + gckVIDMEM_NODE nodeObject = dmabuf->priv; + gcuVIDMEM_NODE_PTR node = nodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + gctINT8_PTR kvaddr = (gctINT8_PTR)ptr - (offset << PAGE_SHIFT); + gctPHYS_ADDR physical = gcvNULL; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + physical = node->VidMem.parent->physical; + kvaddr -= node->VidMem.offset; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + physical = vidMemBlock->physical; + kvaddr -= node->VirtualChunk.offset; + } + else + { + physical = node->Virtual.physical; + } + + gcmkVERIFY_OK(gckOS_DestroyKernelMapping( + nodeObject->kernel->os, physical, (gctPOINTER*)&kvaddr)); +} +#endif + +static struct dma_buf_ops _dmabuf_ops = +{ + .map_dma_buf = _dmabuf_map, + .unmap_dma_buf = _dmabuf_unmap, + .mmap = _dmabuf_mmap, + .release = _dmabuf_release, +#if LINUX_VERSION_CODE > KERNEL_VERSION(5,5,7) +# elif LINUX_VERSION_CODE >= KERNEL_VERSION(4,19,0) + .map = _dmabuf_kmap, + .unmap = _dmabuf_kunmap, +# elif LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + .map_atomic = _dmabuf_kmap, + .unmap_atomic = _dmabuf_kunmap, + .map = _dmabuf_kmap, + .unmap = _dmabuf_kunmap, +# else + .kmap_atomic = _dmabuf_kmap, + .kunmap_atomic = _dmabuf_kunmap, + .kmap = _dmabuf_kmap, + .kunmap = _dmabuf_kunmap, +# endif +}; +#endif + +gceSTATUS +gckVIDMEM_NODE_Export( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + IN gctINT32 Flags, + OUT gctPOINTER *DmaBuf, + OUT gctINT32 *FD + ) +{ +#if defined(CONFIG_DMA_SHARED_BUFFER) + gceSTATUS status = gcvSTATUS_OK; + struct dma_buf *dmabuf = gcvNULL; + + gcmkHEADER_ARG("Kernel=%p NodeObject=0x%x", Kernel, NodeObject); + + dmabuf = NodeObject->dmabuf; + if (!dmabuf) + { + gctSIZE_T bytes = 0; + gctPHYS_ADDR physical = gcvNULL; + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + physical = node->VidMem.parent->physical; + bytes = node->VidMem.bytes; + /* Align export size. when allocate memory from VIDMEM, the actual node size may not same with aligned size. */ + bytes = bytes & ~(PAGE_SIZE - 1); + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + physical = vidMemBlock->physical; + bytes = node->VirtualChunk.bytes; + } + else + { + physical = node->Virtual.physical; + bytes = node->Virtual.bytes; + } + + /* Donot really get SGT, just check if the allocator support GetSGT. */ + gcmkONERROR(gckOS_MemoryGetSGT(Kernel->os, physical, 0, 0, NULL)); + + { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0) + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + exp_info.ops = &_dmabuf_ops; + exp_info.size = bytes; + exp_info.flags = Flags; + exp_info.priv = NodeObject; + dmabuf = dma_buf_export(&exp_info); +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0) + dmabuf = dma_buf_export(NodeObject, &_dmabuf_ops, bytes, Flags, NULL); +#else + dmabuf = dma_buf_export(NodeObject, &_dmabuf_ops, bytes, Flags); +#endif + } + + if (IS_ERR(dmabuf)) + { + gcmkONERROR(gcvSTATUS_GENERIC_IO); + } + + /* Reference this gckVIDMEM_NODE object. */ + gckVIDMEM_NODE_Reference(Kernel, NodeObject); + NodeObject->dmabuf = dmabuf; + } + + if (DmaBuf) + { + *DmaBuf = NodeObject->dmabuf; + } + + if (FD) + { + gctINT fd = dma_buf_fd(dmabuf, Flags); + + if (fd < 0) + { + gcmkONERROR(gcvSTATUS_GENERIC_IO); + } + + *FD = fd; + } + +OnError: + gcmkFOOTER_ARG("*DmaBuf=%p *FD=0x%x", gcmOPT_POINTER(DmaBuf), gcmOPT_VALUE(FD)); + return status; +#else + gcmkFATAL("The kernel did NOT support CONFIG_DMA_SHARED_BUFFER"); + return gcvSTATUS_NOT_SUPPORTED; +#endif +} + +gceSTATUS +gckVIDMEM_NODE_Name( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctUINT32 * Name + ) +{ + gceSTATUS status; + gctUINT32 name = 0; + gctPOINTER database = Kernel->db->nameDatabase; + gctPOINTER mutex = Kernel->db->nameDatabaseMutex; + gctBOOL acquired = gcvFALSE; + gctBOOL referenced = gcvFALSE; + gcmkHEADER_ARG("Kernel=0x%X NodeObject=%p", Kernel, NodeObject); + + gcmkVERIFY_ARGUMENT(Name != gcvNULL); + + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); + acquired = gcvTRUE; + + gcmkONERROR(gckVIDMEM_NODE_Reference(Kernel, NodeObject)); + referenced = gcvTRUE; + + if (NodeObject->name == 0) + { + /* Name this node. */ + gcmkONERROR(gckKERNEL_AllocateIntegerId(database, NodeObject, &name)); + NodeObject->name = name; + } + else + { + name = NodeObject->name; + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + acquired = gcvFALSE; + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, NodeObject)); + + *Name = name; + + gcmkFOOTER_ARG("*Name=%d", *Name); + return gcvSTATUS_OK; + +OnError: + if (referenced) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, NodeObject)); + } + + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_Import( + IN gckKERNEL Kernel, + IN gctUINT32 Name, + OUT gckVIDMEM_NODE * NodeObject + ) +{ + gceSTATUS status; + gckVIDMEM_NODE node = gcvNULL; + gctPOINTER database = Kernel->db->nameDatabase; + gctPOINTER mutex = Kernel->db->nameDatabaseMutex; + gctBOOL acquired = gcvFALSE; + gctBOOL referenced = gcvFALSE; + + gcmkHEADER_ARG("Kernel=0x%X Name=%d", Kernel, Name); + + gcmkONERROR(gckOS_AcquireMutex(Kernel->os, mutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Lookup in database to get the node. */ + gcmkONERROR(gckKERNEL_QueryIntegerId(database, Name, (gctPOINTER *)&node)); + + /* Reference the node. */ + gcmkONERROR(gckVIDMEM_NODE_Reference(Kernel, node)); + referenced = gcvTRUE; + + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + acquired = gcvFALSE; + + *NodeObject = node; + gcmkFOOTER_ARG("*NodeObject=%p", node); + return gcvSTATUS_OK; + +OnError: + if (referenced) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, node)); + } + + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, mutex)); + } + + gcmkFOOTER(); + return status; +} + +typedef struct _gcsVIDMEM_NODE_FDPRIVATE +{ + gcsFDPRIVATE base; + gckKERNEL kernel; + gckVIDMEM_NODE node; +} +gcsVIDMEM_NODE_FDPRIVATE; + + +static gctINT +_ReleaseFdPrivate( + gcsFDPRIVATE_PTR FdPrivate + ) +{ + /* Cast private info. */ + gcsVIDMEM_NODE_FDPRIVATE * private = (gcsVIDMEM_NODE_FDPRIVATE *) FdPrivate; + + gckVIDMEM_NODE_Dereference(private->kernel, private->node); + gckOS_Free(private->kernel->os, private); + + return 0; +} + + +/******************************************************************************* +** +** gckVIDMEM_NODE_GetFd +** +** Attach a gckVIDMEM_NODE object to a native fd. +** +** OUTPUT: +** +** gctUINT32 * Fd +** Pointer to a variable receiving a native fd from os. +*/ +gceSTATUS +gckVIDMEM_NODE_GetFd( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctINT * Fd + ) +{ + gceSTATUS status; + gctBOOL referenced = gcvFALSE; + gcsVIDMEM_NODE_FDPRIVATE * fdPrivate = gcvNULL; + gcmkHEADER_ARG("Kernel=0x%X NodeObject=%d", Kernel, NodeObject); + + /* Reference node object. */ + gcmkVERIFY_OK(gckVIDMEM_NODE_Reference(Kernel, NodeObject)); + referenced = gcvTRUE; + + /* Allocated fd owns a reference. */ + gcmkONERROR(gckOS_Allocate( + Kernel->os, + gcmSIZEOF(gcsVIDMEM_NODE_FDPRIVATE), + (gctPOINTER *)&fdPrivate + )); + + fdPrivate->base.release = _ReleaseFdPrivate; + fdPrivate->kernel = Kernel; + fdPrivate->node = NodeObject; + + /* Allocated fd owns a reference. */ + gcmkONERROR(gckOS_GetFd("vidmem", &fdPrivate->base, Fd)); + + gcmkFOOTER_ARG("*Fd=%d", *Fd); + return gcvSTATUS_OK; + +OnError: + if (referenced) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, NodeObject)); + } + + if (fdPrivate) + { + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, fdPrivate)); + } + + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_WrapUserMemory( + IN gckKERNEL Kernel, + IN gcsUSER_MEMORY_DESC_PTR Desc, + IN gceVIDMEM_TYPE Type, + OUT gckVIDMEM_NODE * NodeObject, + OUT gctUINT64 * Bytes + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gckVIDMEM_NODE nodeObject = gcvNULL; + gctBOOL found = gcvFALSE; + + gcmkHEADER_ARG("Kernel=0x%x", Kernel); + + gcmkVERIFY_ARGUMENT(Desc != gcvNULL); + +#if defined(CONFIG_DMA_SHARED_BUFFER) + if (Desc->flag & gcvALLOC_FLAG_DMABUF) + { + struct dma_buf *dmabuf; + int fd = (int)Desc->handle; + + if (fd >= 0) + { + /* Import dma buf handle. */ + dmabuf = dma_buf_get(fd); + + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + Desc->handle = -1; + Desc->dmabuf = gcmPTR_TO_UINT64(dmabuf); + + dma_buf_put(dmabuf); + } + else + { + if (!Desc->dmabuf) + { + gcmkPRINT("Wrap user memory: invalid dmabuf from user.\n"); + + gcmkFOOTER(); + return gcvSTATUS_INVALID_ARGUMENT; + } + + dmabuf = gcmUINT64_TO_PTR(Desc->dmabuf); + } + + if (dmabuf->ops == &_dmabuf_ops) + { + gctBOOL referenced = gcvFALSE; + nodeObject = dmabuf->priv; + + do + { + /* Reference the node. */ + gcmkERR_BREAK(gckVIDMEM_NODE_Reference(Kernel, nodeObject)); + referenced = gcvTRUE; + found = gcvTRUE; + + *NodeObject = nodeObject; + *Bytes = (gctUINT64)dmabuf->size; + } + while (gcvFALSE); + + if (gcmIS_ERROR(status) && referenced) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference(Kernel, nodeObject)); + } + } + } +#endif + + if (!found) + { + gckOS os = Kernel->os; + gcuVIDMEM_NODE_PTR node = gcvNULL; + + do + { + gctSIZE_T pageCountCpu = 0; + gctSIZE_T pageSizeCpu = 0; + gctPHYS_ADDR_T physicalAddress = 0; + + gcmkVERIFY_OK(gckOS_GetPageSize(os, &pageSizeCpu)); + + /* Allocate an gcuVIDMEM_NODE union. */ + gcmkERR_BREAK(gckOS_Allocate(os, gcmSIZEOF(gcuVIDMEM_NODE), (gctPOINTER*)&node)); + gckOS_ZeroMemory(node, gcmSIZEOF(gcuVIDMEM_NODE)); + + /* Initialize gcuVIDMEM_NODE union for virtual memory. */ + node->Virtual.kernel = Kernel; + + /* Wrap Memory. */ + gcmkERR_BREAK( + gckOS_WrapMemory(os, + Desc, + &node->Virtual.bytes, + &node->Virtual.physical, + &node->Virtual.contiguous, + &pageCountCpu)); + + /* Get base physical address. */ + gcmkERR_BREAK( + gckOS_GetPhysicalFromHandle(os, + node->Virtual.physical, + 0, + &physicalAddress)); + + /* Allocate handle for this video memory. */ + gcmkERR_BREAK(gckVIDMEM_NODE_Construct( + Kernel, + node, + Type, + gcvPOOL_VIRTUAL, + &nodeObject + )); + + node->Virtual.pageCount = (pageCountCpu * pageSizeCpu - + (physicalAddress & (pageSizeCpu - 1) & ~(4096 - 1))) >> 12; + + *NodeObject = nodeObject; + *Bytes = (gctUINT64)node->Virtual.bytes; + } + while (gcvFALSE); + + if (gcmIS_ERROR(status) && node) + { + /* Free the structure. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, node)); + } + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckVIDMEM_NODE_SetCommitStamp( + IN gckKERNEL Kernel, + IN gceENGINE Engine, + IN gckVIDMEM_NODE NodeObject, + IN gctUINT64 CommitStamp + ) +{ + NodeObject->sync[Engine].commitStamp = CommitStamp; + return gcvSTATUS_OK; +} + +gceSTATUS +gckVIDMEM_NODE_GetCommitStamp( + IN gckKERNEL Kernel, + IN gceENGINE Engine, + IN gckVIDMEM_NODE NodeObject, + OUT gctUINT64_PTR CommitStamp + ) +{ + *CommitStamp = NodeObject->sync[Engine].commitStamp; + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckVIDMEM_NODE_Find +** +** Find gckVIDMEM_NODE object according to GPU address of specified core. +** +** INPUT: +** +** gckKERNEL Kernel +** Kernel object, specifies core. +** +** gctUINT32 Address +** GPU address to search. +** +** OUTPUT: +** +** gckVIDMEM_NODE * NodeObject +** Pointer to a variable hold found video memory node. +** +** gctUINT32 * Offset +** The offset of specified GPU address in found video memory node. +*/ +gceSTATUS +gckVIDMEM_NODE_Find( + IN gckKERNEL Kernel, + IN gctUINT32 Address, + OUT gckVIDMEM_NODE * NodeObject, + OUT gctUINT32 * Offset + ) +{ + gceSTATUS status = gcvSTATUS_NOT_FOUND; + gckVIDMEM_NODE nodeObject = gcvNULL; + gcuVIDMEM_NODE_PTR node = gcvNULL; + gckVIDMEM_BLOCK vidMemBlock = gcvNULL; + gcsLISTHEAD_PTR pos; + gceHARDWARE_TYPE hwType; + + gcmkVERIFY_OK( + gckKERNEL_GetHardwareType(Kernel, + &hwType)); + + gcmkVERIFY_OK( + gckOS_AcquireMutex(Kernel->os, + Kernel->db->videoMemListMutex, + gcvINFINITE)); + + gcmkLIST_FOR_EACH(pos, &Kernel->db->videoMemList) + { + nodeObject = (gckVIDMEM_NODE)gcmCONTAINEROF(pos, struct _gcsVIDMEM_NODE, link); + node = nodeObject->node; + vidMemBlock = node->VirtualChunk.parent; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + if (!node->VidMem.locked) + { + /* Don't check against unlocked node. */ + continue; + } + + if (Address >= node->VidMem.address && + Address <= node->VidMem.address + node->VidMem.bytes - 1) + { + *NodeObject = nodeObject; + + if (Offset) + { + *Offset = Address - node->VidMem.address; + } + + status = gcvSTATUS_OK; + break; + } + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + if (!node->VirtualChunk.lockeds[hwType]) + { + /* Don't check against unlocked node. */ + continue; + } + + if (Address >= node->VirtualChunk.addresses[hwType] && + Address <= node->VirtualChunk.addresses[hwType] + node->VirtualChunk.bytes - 1) + { + *NodeObject = nodeObject; + + if (Offset) + { + *Offset = Address - node->VirtualChunk.addresses[hwType]; + } + + status = gcvSTATUS_OK; + break; + } + } + else + { + if (!node->Virtual.lockeds[hwType]) + { + /* Don't check against unlocked node. */ + continue; + } + + if (Address >= node->Virtual.addresses[hwType] && + (Address <= node->Virtual.addresses[hwType] + node->Virtual.bytes - 1)) + { + *NodeObject = nodeObject; + + if (Offset) + { + *Offset = Address - node->Virtual.addresses[hwType]; + } + + status = gcvSTATUS_OK; + break; + } + } + } + + gcmkVERIFY_OK( + gckOS_ReleaseMutex(Kernel->os, Kernel->db->videoMemListMutex)); + + return status; +} + +gceSTATUS +gckVIDMEM_NODE_IsContiguous( + IN gckKERNEL Kernel, + IN gckVIDMEM_NODE NodeObject, + OUT gctBOOL * Contiguous + ) +{ + gceSTATUS status; + gckOS os = Kernel->os; + gctBOOL acquired = gcvFALSE; + gcuVIDMEM_NODE_PTR node = NodeObject->node; + gckVIDMEM_BLOCK vidMemBlock = node->VirtualChunk.parent; + + gcmkHEADER(); + + /* Grab the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(os, NodeObject->mutex, gcvINFINITE)); + acquired = gcvTRUE; + + *Contiguous = gcvFALSE; + + if (node->VidMem.parent->object.type == gcvOBJ_VIDMEM) + { + *Contiguous = gcvTRUE; + } + else if (vidMemBlock && vidMemBlock->object.type == gcvOBJ_VIDMEM_BLOCK) + { + if (vidMemBlock->contiguous) + { + *Contiguous = gcvTRUE; + } + } + else + { + if (node->Virtual.contiguous) + { + *Contiguous = gcvTRUE; + } + } + + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(os, NodeObject->mutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +}
diff --git a/hal/kernel/inc/gc_hal.h b/hal/kernel/inc/gc_hal.h new file mode 100644 index 0000000..231a80d --- /dev/null +++ b/hal/kernel/inc/gc_hal.h
@@ -0,0 +1,2159 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_h_ +#define __gc_hal_h_ + +#include "shared/gc_hal_types.h" +#include "gc_hal_enum.h" +#include "gc_hal_base.h" +#include "gc_hal_profiler.h" +#include "shared/gc_hal_driver.h" +#if gcdENABLE_3D +#include "gc_hal_statistics.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/******************************************************************************\ +******************************* Alignment Macros ******************************* +\******************************************************************************/ + +/* Alignment with a non-power of two value. */ +#define gcmALIGN_NP2(n, align) (((n) + (align) - 1) - (((n) + (align) - 1) % (align))) + +#define gcmALIGN_NP2_SAFE(n, align) \ +(\ + (gcmALIGN_NP2((n) & ~0ULL, (align) & ~0ULL) ^ gcmALIGN_NP2(n, align)) ? \ + (n) : gcmALIGN_NP2(n, align) \ +) + +/* Alignment with a power of two value. */ +#define gcmALIGN(n, align) (((n) + ((align) - 1)) & ~((align) - 1)) + +#define gcmALIGN_SAFE(n, align) \ +(\ + (gcmALIGN((n) & ~0ULL, (align) & ~0ULL) ^ gcmALIGN(n, align)) ? \ + (n) : gcmALIGN(n, align) \ +) + +#define gcmALIGN_BASE(n, align) \ +(\ + ((n) & ~((align) - 1)) \ +) + +/******************************************************************************\ +***************************** Element Count Macro ***************************** +\******************************************************************************/ + +#define gcmSIZEOF(a) \ +(\ + (gctSIZE_T) (sizeof(a)) \ +) + +#define gcmCOUNTOF(a) \ +(\ + sizeof(a) / sizeof(a[0]) \ +) + +/******************************************************************************\ +********************************* Cast Macro ********************************** +\******************************************************************************/ +#define gcmNAME_TO_PTR(na) \ + gckKERNEL_QueryPointerFromName(kernel, gcmALL_TO_UINT32(na)) + +#define gcmPTR_TO_NAME(ptr) \ + gckKERNEL_AllocateNameFromPointer(kernel, ptr) + +#define gcmRELEASE_NAME(na) \ + gckKERNEL_DeleteName(kernel, gcmALL_TO_UINT32(na)) + +#define gcmALL_TO_UINT32(t) \ +(\ + (gctUINT32) (gctUINTPTR_T) (t)\ +) + +#define gcmPTR_TO_UINT64(p) \ +(\ + (gctUINT64) (gctUINTPTR_T) (p)\ +) + +#define gcmUINT64_TO_PTR(u) \ +(\ + (gctPOINTER) (gctUINTPTR_T) (u)\ +) + +#define gcmUINT64_TO_TYPE(u, t) \ +(\ + (t) (gctUINTPTR_T) (u)\ +) + +/******************************************************************************\ +******************************** Useful Macro ********************************* +\******************************************************************************/ + +#define gcvINVALID_ADDRESS ~0U +#define gcvINVALID_VALUE 0xCCCCCCCC + +#define gcvINVALID_PHYSICAL_ADDRESS ~0ULL + +#define gcmGET_PRE_ROTATION(rotate) \ + ((rotate) & (~(gcvSURF_POST_FLIP_X | gcvSURF_POST_FLIP_Y))) + +#define gcmGET_POST_ROTATION(rotate) \ + ((rotate) & (gcvSURF_POST_FLIP_X | gcvSURF_POST_FLIP_Y)) + +typedef struct _gckHARDWARE * gckHARDWARE; + +#define gcdMAX_GPU_COUNT gcvCORE_COUNT + +#define gcdMAX_SURF_LAYERS 4 + +#define gcdMAX_DRAW_BUFFERS 16 + +#define gcdMAX_3DGPU_COUNT 8 + +#define gcdMAX_MAJOR_CORE_COUNT 8 + +#define gcdMAX_VERTEX_STREAM_COUNT 4 +/******************************************************************************* +** +** gcmVERIFY_OBJECT +** +** Assert if an object is invalid or is not of the specified type. If the +** object is invalid or not of the specified type, gcvSTATUS_INVALID_OBJECT +** will be returned from the current function. In retail mode this macro +** does nothing. +** +** ARGUMENTS: +** +** obj Object to test. +** t Expected type of the object. +*/ +#if gcmIS_DEBUG(gcdDEBUG_TRACE) +#define _gcmVERIFY_OBJECT(prefix, obj, t) \ + if ((obj) == gcvNULL) \ + { \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "VERIFY_OBJECT failed: NULL"); \ + prefix##TRACE(gcvLEVEL_ERROR, " expected: %c%c%c%c", \ + gcmCC_PRINT(t)); \ + prefix##ASSERT((obj) != gcvNULL); \ + prefix##FOOTER_ARG("status=%d", gcvSTATUS_INVALID_OBJECT); \ + return gcvSTATUS_INVALID_OBJECT; \ + } \ + else if (((gcsOBJECT*) (obj))->type != t) \ + { \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "VERIFY_OBJECT failed: %c%c%c%c", \ + gcmCC_PRINT(((gcsOBJECT*) (obj))->type)); \ + prefix##TRACE(gcvLEVEL_ERROR, " expected: %c%c%c%c", \ + gcmCC_PRINT(t)); \ + prefix##ASSERT(((gcsOBJECT*)(obj))->type == t); \ + prefix##FOOTER_ARG("status=%d", gcvSTATUS_INVALID_OBJECT); \ + return gcvSTATUS_INVALID_OBJECT; \ + } + +# define gcmVERIFY_OBJECT(obj, t) _gcmVERIFY_OBJECT(gcm, obj, t) +# define gcmkVERIFY_OBJECT(obj, t) _gcmVERIFY_OBJECT(gcmk, obj, t) +#else +# define gcmVERIFY_OBJECT(obj, t) do {} while (gcvFALSE) +# define gcmkVERIFY_OBJECT(obj, t) do {} while (gcvFALSE) +#endif + +/******************************************************************************/ +/*VERIFY_OBJECT if special return expected*/ +/******************************************************************************/ +#ifndef EGL_API_ANDROID +# define _gcmVERIFY_OBJECT_RETURN(prefix, obj, t, retVal) \ + do \ + { \ + if ((obj) == gcvNULL) \ + { \ + prefix##PRINT_VERSION(); \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "VERIFY_OBJECT_RETURN failed: NULL"); \ + prefix##TRACE(gcvLEVEL_ERROR, " expected: %c%c%c%c", \ + gcmCC_PRINT(t)); \ + prefix##ASSERT((obj) != gcvNULL); \ + prefix##FOOTER_ARG("retVal=%d", retVal); \ + return retVal; \ + } \ + else if (((gcsOBJECT*) (obj))->type != t) \ + { \ + prefix##PRINT_VERSION(); \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "VERIFY_OBJECT_RETURN failed: %c%c%c%c", \ + gcmCC_PRINT(((gcsOBJECT*) (obj))->type)); \ + prefix##TRACE(gcvLEVEL_ERROR, " expected: %c%c%c%c", \ + gcmCC_PRINT(t)); \ + prefix##ASSERT(((gcsOBJECT*)(obj))->type == t); \ + prefix##FOOTER_ARG("retVal=%d", retVal); \ + return retVal; \ + } \ + } \ + while (gcvFALSE) +# define gcmVERIFY_OBJECT_RETURN(obj, t, retVal) \ + _gcmVERIFY_OBJECT_RETURN(gcm, obj, t, retVal) +# define gcmkVERIFY_OBJECT_RETURN(obj, t, retVal) \ + _gcmVERIFY_OBJECT_RETURN(gcmk, obj, t, retVal) +#else +# define gcmVERIFY_OBJECT_RETURN(obj, t) do {} while (gcvFALSE) +# define gcmVERIFY_OBJECT_RETURN(obj, t) do {} while (gcvFALSE) +#endif + +/******************************************************************************\ +********************************** gckOS Object ********************************* +\******************************************************************************/ + +/* Construct a new gckOS object. */ +gceSTATUS +gckOS_Construct( + IN gctPOINTER Context, + OUT gckOS * Os + ); + +/* Destroy an gckOS object. */ +gceSTATUS +gckOS_Destroy( + IN gckOS Os + ); + +/* Query the video memory. */ +gceSTATUS +gckOS_QueryVideoMemory( + IN gckOS Os, + OUT gctPHYS_ADDR * InternalAddress, + OUT gctSIZE_T * InternalSize, + OUT gctPHYS_ADDR * ExternalAddress, + OUT gctSIZE_T * ExternalSize, + OUT gctPHYS_ADDR * ContiguousAddress, + OUT gctSIZE_T * ContiguousSize + ); + +/* Allocate memory from the heap. */ +gceSTATUS +gckOS_Allocate( + IN gckOS Os, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Memory + ); + +/* Free allocated memory. */ +gceSTATUS +gckOS_Free( + IN gckOS Os, + IN gctPOINTER Memory + ); + +/* Wrapper for allocation memory.. */ +gceSTATUS +gckOS_AllocateMemory( + IN gckOS Os, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Memory + ); + +/* Wrapper for freeing memory. */ +gceSTATUS +gckOS_FreeMemory( + IN gckOS Os, + IN gctPOINTER Memory + ); + +/* Allocate paged memory. */ +gceSTATUS +gckOS_AllocatePagedMemory( + IN gckOS Os, + IN gctUINT32 Flag, + IN OUT gctSIZE_T * Bytes, + OUT gctUINT32 * Gid, + OUT gctPHYS_ADDR * Physical + ); + +/* Lock pages. */ +gceSTATUS +gckOS_LockPages( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Bytes, + IN gctBOOL Cacheable, + OUT gctPOINTER * Logical + ); + +/* Map pages. */ +gceSTATUS +gckOS_MapPagesEx( + IN gckOS Os, + IN gceCORE Core, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T PageCount, + IN gctUINT32 Address, + IN gctPOINTER PageTable, + IN gctBOOL Writable, + IN gceVIDMEM_TYPE Type + ); + +/* Map 1M pages. */ +gceSTATUS +gckOS_Map1MPages( + IN gckOS Os, + IN gceCORE Core, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T PageCount, + IN gctUINT32 Address, + IN gctPOINTER PageTable, + IN gctBOOL Writable, + IN gceVIDMEM_TYPE Type + ); + +gceSTATUS +gckOS_UnmapPages( + IN gckOS Os, + IN gctSIZE_T PageCount, + IN gctUINT32 Address + ); + +/* Unlock pages. */ +gceSTATUS +gckOS_UnlockPages( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Bytes, + IN gctPOINTER Logical + ); + +/* Free paged memory. */ +gceSTATUS +gckOS_FreePagedMemory( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Bytes + ); + +/* Allocate non-paged memory. */ +gceSTATUS +gckOS_AllocateNonPagedMemory( + IN gckOS Os, + IN gctBOOL InUserSpace, + IN gctUINT32 Flag, + IN OUT gctSIZE_T * Bytes, + OUT gctPHYS_ADDR * Physical, + OUT gctPOINTER * Logical + ); + +/* Free non-paged memory. */ +gceSTATUS +gckOS_FreeNonPagedMemory( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ); + +/* Reserved memory. */ +gceSTATUS +gckOS_RequestReservedMemory( + gckOS Os, + gctPHYS_ADDR_T Start, + gctSIZE_T Size, + const char * Name, + gctBOOL Requested, + gctPOINTER * MemoryHandle + ); + +void +gckOS_ReleaseReservedMemory( + gckOS Os, + gctPOINTER MemoryHandle + ); + +/* Get the number fo bytes per page. */ +gceSTATUS +gckOS_GetPageSize( + IN gckOS Os, + OUT gctSIZE_T * PageSize + ); + +/* Get the physical address of a corresponding logical address. */ +gceSTATUS +gckOS_GetPhysicalAddress( + IN gckOS Os, + IN gctPOINTER Logical, + OUT gctPHYS_ADDR_T * Address + ); + +/* Get real physical address from handle. */ +gceSTATUS +gckOS_GetPhysicalFromHandle( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctUINT32 Offset, + OUT gctPHYS_ADDR_T * PhysicalAddress + ); + +/* Get the physical address of a corresponding user logical address. */ +gceSTATUS +gckOS_UserLogicalToPhysical( + IN gckOS Os, + IN gctPOINTER Logical, + OUT gctPHYS_ADDR_T * Address + ); + +/* Map physical memory. */ +gceSTATUS +gckOS_MapPhysical( + IN gckOS Os, + IN gctPHYS_ADDR_T Physical, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Logical + ); + +/* Unmap previously mapped physical memory. */ +gceSTATUS +gckOS_UnmapPhysical( + IN gckOS Os, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ); + +/* Read data from a hardware register. */ +gceSTATUS +gckOS_ReadRegister( + IN gckOS Os, + IN gctUINT32 Address, + OUT gctUINT32 * Data + ); + +/* Read data from a hardware register. */ +gceSTATUS +gckOS_ReadRegisterEx( + IN gckOS Os, + IN gceCORE Core, + IN gctUINT32 Address, + OUT gctUINT32 * Data + ); + +/* Write data to a hardware register. */ +gceSTATUS +gckOS_WriteRegister( + IN gckOS Os, + IN gctUINT32 Address, + IN gctUINT32 Data + ); + +/* Write data to a hardware register. */ +gceSTATUS +gckOS_WriteRegisterEx( + IN gckOS Os, + IN gceCORE Core, + IN gctUINT32 Address, + IN gctUINT32 Data + ); + +/* Write data to a hardware register without dump. */ +gceSTATUS +gckOS_WriteRegisterEx_NoDump( + IN gckOS Os, + IN gceCORE Core, + IN gctUINT32 Address, + IN gctUINT32 Data + ); + +#ifdef __QNXNTO__ +static gcmINLINE gceSTATUS +gckOS_WriteMemory( + IN gckOS Os, + IN gctPOINTER Address, + IN gctUINT32 Data + ) +{ + /* Write memory. */ + *(gctUINT32 *)Address = Data; + return gcvSTATUS_OK; +} + +#else +/* Write data to a 32-bit memory location. */ +gceSTATUS +gckOS_WriteMemory( + IN gckOS Os, + IN gctPOINTER Address, + IN gctUINT32 Data + ); +#endif + +/* Map physical memory into the process space. */ +gceSTATUS +gckOS_MapMemory( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Logical + ); + +/* Unmap physical memory from the specified process space. */ +gceSTATUS +gckOS_UnmapMemoryEx( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Bytes, + IN gctPOINTER Logical, + IN gctUINT32 PID + ); + +/* Unmap physical memory from the process space. */ +gceSTATUS +gckOS_UnmapMemory( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Bytes, + IN gctPOINTER Logical + ); + +/* Delete a mutex. */ +gceSTATUS +gckOS_DeleteMutex( + IN gckOS Os, + IN gctPOINTER Mutex + ); + +/* Acquire a mutex. */ +gceSTATUS +gckOS_AcquireMutex( + IN gckOS Os, + IN gctPOINTER Mutex, + IN gctUINT32 Timeout + ); + +/* Release a mutex. */ +gceSTATUS +gckOS_ReleaseMutex( + IN gckOS Os, + IN gctPOINTER Mutex + ); + +/* Atomically exchange a pair of 32-bit values. */ +gceSTATUS +gckOS_AtomicExchange( + IN gckOS Os, + IN OUT gctUINT32_PTR Target, + IN gctUINT32 NewValue, + OUT gctUINT32_PTR OldValue + ); + +/* Atomically exchange a pair of pointers. */ +gceSTATUS +gckOS_AtomicExchangePtr( + IN gckOS Os, + IN OUT gctPOINTER * Target, + IN gctPOINTER NewValue, + OUT gctPOINTER * OldValue + ); + +gceSTATUS +gckOS_AtomSetMask( + IN gctPOINTER Atom, + IN gctUINT32 Mask + ); + +gceSTATUS +gckOS_AtomClearMask( + IN gctPOINTER Atom, + IN gctUINT32 Mask + ); + +gceSTATUS +gckOS_DumpCallStack( + IN gckOS Os + ); + +gceSTATUS +gckOS_GetProcessNameByPid( + IN gctINT Pid, + IN gctSIZE_T Length, + OUT gctUINT8_PTR String + ); + +/******************************************************************************* +** +** gckOS_AtomConstruct +** +** Create an atom. +** +** INPUT: +** +** gckOS Os +** Pointer to a gckOS object. +** +** OUTPUT: +** +** gctPOINTER * Atom +** Pointer to a variable receiving the constructed atom. +*/ +gceSTATUS +gckOS_AtomConstruct( + IN gckOS Os, + OUT gctPOINTER * Atom + ); + +/******************************************************************************* +** +** gckOS_AtomDestroy +** +** Destroy an atom. +** +** INPUT: +** +** gckOS Os +** Pointer to a gckOS object. +** +** gctPOINTER Atom +** Pointer to the atom to destroy. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckOS_AtomDestroy( + IN gckOS Os, + OUT gctPOINTER Atom + ); + +/******************************************************************************* +** +** gckOS_AtomGet +** +** Get the 32-bit value protected by an atom. +** +** INPUT: +** +** gckOS Os +** Pointer to a gckOS object. +** +** gctPOINTER Atom +** Pointer to the atom. +** +** OUTPUT: +** +** gctINT32_PTR Value +** Pointer to a variable the receives the value of the atom. +*/ +gceSTATUS +gckOS_AtomGet( + IN gckOS Os, + IN gctPOINTER Atom, + OUT gctINT32_PTR Value + ); + +/******************************************************************************* +** +** gckOS_AtomSet +** +** Set the 32-bit value protected by an atom. +** +** INPUT: +** +** gckOS Os +** Pointer to a gckOS object. +** +** gctPOINTER Atom +** Pointer to the atom. +** +** gctINT32 Value +** The value of the atom. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckOS_AtomSet( + IN gckOS Os, + IN gctPOINTER Atom, + IN gctINT32 Value + ); + +/******************************************************************************* +** +** gckOS_AtomIncrement +** +** Atomically increment the 32-bit integer value inside an atom. +** +** INPUT: +** +** gckOS Os +** Pointer to a gckOS object. +** +** gctPOINTER Atom +** Pointer to the atom. +** +** OUTPUT: +** +** gctINT32_PTR Value +** Pointer to a variable the receives the original value of the atom. +*/ +gceSTATUS +gckOS_AtomIncrement( + IN gckOS Os, + IN gctPOINTER Atom, + OUT gctINT32_PTR Value + ); + +/******************************************************************************* +** +** gckOS_AtomDecrement +** +** Atomically decrement the 32-bit integer value inside an atom. +** +** INPUT: +** +** gckOS Os +** Pointer to a gckOS object. +** +** gctPOINTER Atom +** Pointer to the atom. +** +** OUTPUT: +** +** gctINT32_PTR Value +** Pointer to a variable the receives the original value of the atom. +*/ +gceSTATUS +gckOS_AtomDecrement( + IN gckOS Os, + IN gctPOINTER Atom, + OUT gctINT32_PTR Value + ); + +/* Delay a number of milliseconds. */ +gceSTATUS +gckOS_Delay( + IN gckOS Os, + IN gctUINT32 Delay + ); + +/* Get time in milliseconds. */ +gceSTATUS +gckOS_GetTicks( + OUT gctUINT32_PTR Time + ); + +/* Compare time value. */ +gceSTATUS +gckOS_TicksAfter( + IN gctUINT32 Time1, + IN gctUINT32 Time2, + OUT gctBOOL_PTR IsAfter + ); + +/* Get time in microseconds. */ +gceSTATUS +gckOS_GetTime( + OUT gctUINT64_PTR Time + ); + +/* Memory barrier. */ +gceSTATUS +gckOS_MemoryBarrier( + IN gckOS Os, + IN gctPOINTER Address + ); + +/* Map user pointer. */ +gceSTATUS +gckOS_MapUserPointer( + IN gckOS Os, + IN gctPOINTER Pointer, + IN gctSIZE_T Size, + OUT gctPOINTER * KernelPointer + ); + +/* Unmap user pointer. */ +gceSTATUS +gckOS_UnmapUserPointer( + IN gckOS Os, + IN gctPOINTER Pointer, + IN gctSIZE_T Size, + IN gctPOINTER KernelPointer + ); + +/******************************************************************************* +** +** gckOS_QueryNeedCopy +** +** Query whether the memory can be accessed or mapped directly or it has to be +** copied. +** +** INPUT: +** +** gckOS Os +** Pointer to an gckOS object. +** +** gctUINT32 ProcessID +** Process ID of the current process. +** +** OUTPUT: +** +** gctBOOL_PTR NeedCopy +** Pointer to a boolean receiving gcvTRUE if the memory needs a copy or +** gcvFALSE if the memory can be accessed or mapped dircetly. +*/ +gceSTATUS +gckOS_QueryNeedCopy( + IN gckOS Os, + IN gctUINT32 ProcessID, + OUT gctBOOL_PTR NeedCopy + ); + +/******************************************************************************* +** +** gckOS_CopyFromUserData +** +** Copy data from user to kernel memory. +** +** INPUT: +** +** gckOS Os +** Pointer to an gckOS object. +** +** gctPOINTER KernelPointer +** Pointer to kernel memory. +** +** gctPOINTER Pointer +** Pointer to user memory. +** +** gctSIZE_T Size +** Number of bytes to copy. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckOS_CopyFromUserData( + IN gckOS Os, + IN gctPOINTER KernelPointer, + IN gctPOINTER Pointer, + IN gctSIZE_T Size + ); + +/******************************************************************************* +** +** gckOS_CopyToUserData +** +** Copy data from kernel to user memory. +** +** INPUT: +** +** gckOS Os +** Pointer to an gckOS object. +** +** gctPOINTER KernelPointer +** Pointer to kernel memory. +** +** gctPOINTER Pointer +** Pointer to user memory. +** +** gctSIZE_T Size +** Number of bytes to copy. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckOS_CopyToUserData( + IN gckOS Os, + IN gctPOINTER KernelPointer, + IN gctPOINTER Pointer, + IN gctSIZE_T Size + ); + +gceSTATUS +gckOS_SuspendInterrupt( + IN gckOS Os + ); + +gceSTATUS +gckOS_SuspendInterruptEx( + IN gckOS Os, + IN gceCORE Core + ); + +gceSTATUS +gckOS_ResumeInterrupt( + IN gckOS Os + ); + +gceSTATUS +gckOS_ResumeInterruptEx( + IN gckOS Os, + IN gceCORE Core + ); + +/* Get the base address for the physical memory. */ +gceSTATUS +gckOS_GetBaseAddress( + IN gckOS Os, + OUT gctUINT32_PTR BaseAddress + ); + +/* Perform a memory copy. */ +gceSTATUS +gckOS_MemCopy( + IN gctPOINTER Destination, + IN gctCONST_POINTER Source, + IN gctSIZE_T Bytes + ); + +/* Zero memory. */ +gceSTATUS +gckOS_ZeroMemory( + IN gctPOINTER Memory, + IN gctSIZE_T Bytes + ); + +/******************************************************************************* +** +** gckOS_GetProcessID +** +** Get current process ID. +** +** INPUT: +** +** Nothing. +** +** OUTPUT: +** +** gctUINT32_PTR ProcessID +** Pointer to the variable that receives the process ID. +*/ +gceSTATUS +gckOS_GetProcessID( + OUT gctUINT32_PTR ProcessID + ); + +gceSTATUS +gckOS_GetCurrentProcessID( + OUT gctUINT32_PTR ProcessID + ); + +/******************************************************************************* +** +** gckOS_GetThreadID +** +** Get current thread ID. +** +** INPUT: +** +** Nothing. +** +** OUTPUT: +** +** gctUINT32_PTR ThreadID +** Pointer to the variable that receives the thread ID. +*/ +gceSTATUS +gckOS_GetThreadID( + OUT gctUINT32_PTR ThreadID + ); + +/******************************************************************************\ +********************************** Signal Object ********************************* +\******************************************************************************/ + +/* Create a signal. */ +gceSTATUS +gckOS_CreateSignal( + IN gckOS Os, + IN gctBOOL ManualReset, + OUT gctSIGNAL * Signal + ); + +/* Destroy a signal. */ +gceSTATUS +gckOS_DestroySignal( + IN gckOS Os, + IN gctSIGNAL Signal + ); + +/* Signal a signal. */ +gceSTATUS +gckOS_Signal( + IN gckOS Os, + IN gctSIGNAL Signal, + IN gctBOOL State + ); + +/* Wait for a signal. */ +gceSTATUS +gckOS_WaitSignal( + IN gckOS Os, + IN gctSIGNAL Signal, + IN gctBOOL Interruptable, + IN gctUINT32 Wait + ); + +#ifdef __QNXNTO__ +gceSTATUS +gckOS_SignalPulse( + IN gckOS Os, + IN gctSIGNAL Signal + ); + +gceSTATUS +gckOS_SignalPending( + IN gckOS Os, + IN gctSIGNAL Signal + ); +#endif + +/* Map a user signal to the kernel space. */ +gceSTATUS +gckOS_MapSignal( + IN gckOS Os, + IN gctSIGNAL Signal, + IN gctHANDLE Process, + OUT gctSIGNAL * MappedSignal + ); + +/* Unmap a user signal */ +gceSTATUS +gckOS_UnmapSignal( + IN gckOS Os, + IN gctSIGNAL Signal + ); + +/* Get scatter-gather table from memory. */ +gceSTATUS +gckOS_MemoryGetSGT( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Offset, + IN gctSIZE_T Bytes, + OUT gctPOINTER *SGT + ); + +/* Map a page range of memory to user space. */ +gceSTATUS +gckOS_MemoryMmap( + IN gckOS Os, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T skipPages, + IN gctSIZE_T numPages, + INOUT gctPOINTER Vma + ); + +/* Wrap a user memory to gctPHYS_ADDR. */ +gceSTATUS +gckOS_WrapMemory( + IN gckOS Os, + IN gcsUSER_MEMORY_DESC_PTR Desc, + OUT gctSIZE_T *Bytes, + OUT gctPHYS_ADDR * Physical, + OUT gctBOOL *Contiguous, + OUT gctSIZE_T * PageCountCpu + ); + +gceSTATUS +gckOS_GetPolicyID( + IN gckOS Os, + IN gceVIDMEM_TYPE Type, + OUT gctUINT32_PTR PolicyID, + OUT gctUINT32_PTR AXIConfig + ); + +/******************************************************************************\ +************************** Android Native Fence Sync *************************** +\******************************************************************************/ +gceSTATUS +gckOS_CreateSyncTimeline( + IN gckOS Os, + IN gceCORE Core, + OUT gctHANDLE * Timeline + ); + +gceSTATUS +gckOS_DestroySyncTimeline( + IN gckOS Os, + IN gctHANDLE Timeline + ); + +gceSTATUS +gckOS_CreateNativeFence( + IN gckOS Os, + IN gctHANDLE Timeline, + IN gctSIGNAL Signal, + OUT gctINT * FenceFD + ); + +gceSTATUS +gckOS_WaitNativeFence( + IN gckOS Os, + IN gctHANDLE Timeline, + IN gctINT FenceFD, + IN gctUINT32 Timeout + ); + +#if !USE_NEW_LINUX_SIGNAL +/* Create signal to be used in the user space. */ +gceSTATUS +gckOS_CreateUserSignal( + IN gckOS Os, + IN gctBOOL ManualReset, + OUT gctINT * SignalID + ); + +/* Destroy signal used in the user space. */ +gceSTATUS +gckOS_DestroyUserSignal( + IN gckOS Os, + IN gctINT SignalID + ); + +/* Wait for signal used in the user space. */ +gceSTATUS +gckOS_WaitUserSignal( + IN gckOS Os, + IN gctINT SignalID, + IN gctUINT32 Wait + ); + +/* Signal a signal used in the user space. */ +gceSTATUS +gckOS_SignalUserSignal( + IN gckOS Os, + IN gctINT SignalID, + IN gctBOOL State + ); +#endif /* USE_NEW_LINUX_SIGNAL */ + +/* Set a signal owned by a process. */ +#if defined(__QNXNTO__) +gceSTATUS +gckOS_UserSignal( + IN gckOS Os, + IN gctSIGNAL Signal, + IN gctINT Recvid, + IN gctINT Coid + ); +#else +gceSTATUS +gckOS_UserSignal( + IN gckOS Os, + IN gctSIGNAL Signal, + IN gctHANDLE Process + ); +#endif + +/******************************************************************************\ +** Cache Support +*/ + +gceSTATUS +gckOS_CacheClean( + gckOS Os, + gctUINT32 ProcessID, + gctPHYS_ADDR Handle, + gctSIZE_T Offset, + gctPOINTER Logical, + gctSIZE_T Bytes + ); + +gceSTATUS +gckOS_CacheFlush( + gckOS Os, + gctUINT32 ProcessID, + gctPHYS_ADDR Handle, + gctSIZE_T Offset, + gctPOINTER Logical, + gctSIZE_T Bytes + ); + +gceSTATUS +gckOS_CacheInvalidate( + gckOS Os, + gctUINT32 ProcessID, + gctPHYS_ADDR Handle, + gctSIZE_T Offset, + gctPOINTER Logical, + gctSIZE_T Bytes + ); + +gceSTATUS +gckOS_CPUPhysicalToGPUPhysical( + IN gckOS Os, + IN gctPHYS_ADDR_T CPUPhysical, + IN gctPHYS_ADDR_T * GPUPhysical + ); + +gceSTATUS +gckOS_GPUPhysicalToCPUPhysical( + IN gckOS Os, + IN gctUINT32 GPUPhysical, + IN gctPHYS_ADDR_T * CPUPhysical + ); + +gceSTATUS +gckOS_QueryOption( + IN gckOS Os, + IN gctCONST_STRING Option, + OUT gctUINT64 * Value + ); + +/******************************************************************************\ +** Debug Support +*/ + +void +gckOS_SetDebugLevel( + IN gctUINT32 Level + ); + +void +gckOS_SetDebugZone( + IN gctUINT32 Zone + ); + +void +gckOS_SetDebugLevelZone( + IN gctUINT32 Level, + IN gctUINT32 Zone + ); + +void +gckOS_SetDebugZones( + IN gctUINT32 Zones, + IN gctBOOL Enable + ); + +void +gckOS_SetDebugFile( + IN gctCONST_STRING FileName + ); + +gceSTATUS +gckOS_Broadcast( + IN gckOS Os, + IN gckHARDWARE Hardware, + IN gceBROADCAST Reason + ); + +gceSTATUS +gckOS_BroadcastHurry( + IN gckOS Os, + IN gckHARDWARE Hardware, + IN gctUINT Urgency + ); + +gceSTATUS +gckOS_BroadcastCalibrateSpeed( + IN gckOS Os, + IN gckHARDWARE Hardware, + IN gctUINT Idle, + IN gctUINT Time + ); + +/******************************************************************************* +** +** gckOS_SetGPUPower +** +** Set the power of the GPU on or off. +** +** INPUT: +** +** gckOS Os +** Pointer to a gckOS object. +** +** gceCORE Core +** GPU whose power is set. +** +** gctBOOL Clock +** gcvTRUE to turn on the clock, or gcvFALSE to turn off the clock. +** +** gctBOOL Power +** gcvTRUE to turn on the power, or gcvFALSE to turn off the power. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckOS_SetGPUPower( + IN gckOS Os, + IN gceCORE Core, + IN gctBOOL Clock, + IN gctBOOL Power + ); + +gceSTATUS +gckOS_ResetGPU( + IN gckOS Os, + IN gceCORE Core + ); + +gceSTATUS +gckOS_PrepareGPUFrequency( + IN gckOS Os, + IN gceCORE Core + ); + +gceSTATUS +gckOS_FinishGPUFrequency( + IN gckOS Os, + IN gceCORE Core + ); + +gceSTATUS +gckOS_QueryGPUFrequency( + IN gckOS Os, + IN gceCORE Core, + OUT gctUINT32 * Frequency, + OUT gctUINT8 * Scale + ); + +gceSTATUS +gckOS_SetGPUFrequency( + IN gckOS Os, + IN gceCORE Core, + IN gctUINT8 Scale + ); + +/******************************************************************************* +** Semaphores. +*/ + +/* Create a new semaphore. */ +gceSTATUS +gckOS_CreateSemaphore( + IN gckOS Os, + OUT gctPOINTER * Semaphore + ); + + +/* Delete a semahore. */ +gceSTATUS +gckOS_DestroySemaphore( + IN gckOS Os, + IN gctPOINTER Semaphore + ); + +/* Acquire a semahore. */ +gceSTATUS +gckOS_AcquireSemaphore( + IN gckOS Os, + IN gctPOINTER Semaphore + ); + +/* Try to acquire a semahore. */ +gceSTATUS +gckOS_TryAcquireSemaphore( + IN gckOS Os, + IN gctPOINTER Semaphore + ); + +/* Release a semahore. */ +gceSTATUS +gckOS_ReleaseSemaphore( + IN gckOS Os, + IN gctPOINTER Semaphore + ); + +/******************************************************************************* +** Timer API. +*/ + +typedef void (*gctTIMERFUNCTION)(gctPOINTER); + +/* Create a timer. */ +gceSTATUS +gckOS_CreateTimer( + IN gckOS Os, + IN gctTIMERFUNCTION Function, + IN gctPOINTER Data, + OUT gctPOINTER * Timer + ); + +/* Destory a timer. */ +gceSTATUS +gckOS_DestroyTimer( + IN gckOS Os, + IN gctPOINTER Timer + ); + +/* Start a timer. */ +gceSTATUS +gckOS_StartTimer( + IN gckOS Os, + IN gctPOINTER Timer, + IN gctUINT32 Delay + ); + +/* Stop a timer. */ +gceSTATUS +gckOS_StopTimer( + IN gckOS Os, + IN gctPOINTER Timer + ); + +/******************************************************************************\ +********************************* gckHEAP Object ******************************** +\******************************************************************************/ + +typedef struct _gckHEAP * gckHEAP; + +/* Construct a new gckHEAP object. */ +gceSTATUS +gckHEAP_Construct( + IN gckOS Os, + IN gctSIZE_T AllocationSize, + OUT gckHEAP * Heap + ); + +/* Destroy an gckHEAP object. */ +gceSTATUS +gckHEAP_Destroy( + IN gckHEAP Heap + ); + +/* Allocate memory. */ +gceSTATUS +gckHEAP_Allocate( + IN gckHEAP Heap, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Node + ); + +/* Free memory. */ +gceSTATUS +gckHEAP_Free( + IN gckHEAP Heap, + IN gctPOINTER Node + ); + +/* Profile the heap. */ +gceSTATUS +gckHEAP_ProfileStart( + IN gckHEAP Heap + ); + +gceSTATUS +gckHEAP_ProfileEnd( + IN gckHEAP Heap, + IN gctCONST_STRING Title + ); + +typedef struct _gckVIDMEM * gckVIDMEM; +typedef struct _gckKERNEL * gckKERNEL; +typedef struct _gckDB * gckDB; +typedef struct _gckDVFS * gckDVFS; +typedef struct _gckMMU * gckMMU; +typedef struct _gcsDEVICE * gckDEVICE; + +/******************************************************************************\ +******************************** gckKERNEL Object ****************************** +\******************************************************************************/ + +struct _gcsHAL_INTERFACE; + +/* Construct a new gckKERNEL object. */ +gceSTATUS +gckKERNEL_Construct( + IN gckOS Os, + IN gceCORE Core, + IN gctUINT ChipID, + IN gctPOINTER Context, + IN gckDEVICE Device, + IN gckDB SharedDB, + OUT gckKERNEL * Kernel + ); + +/* Destroy an gckKERNEL object. */ +gceSTATUS +gckKERNEL_Destroy( + IN gckKERNEL Kernel + ); + +/* Dispatch a user-level command. */ +gceSTATUS +gckKERNEL_Dispatch( + IN gckKERNEL Kernel, + IN gckDEVICE Device, + IN OUT struct _gcsHAL_INTERFACE * Interface + ); + +/* Query Database requirements. */ +gceSTATUS + gckKERNEL_QueryDatabase( + IN gckKERNEL Kernel, + IN gctUINT32 ProcessID, + IN OUT gcsHAL_INTERFACE * Interface + ); + +/* Query the video memory. */ +gceSTATUS +gckKERNEL_QueryVideoMemory( + IN gckKERNEL Kernel, + OUT struct _gcsHAL_INTERFACE * Interface + ); + +/* Lookup the gckVIDMEM object for a pool. */ +gceSTATUS +gckKERNEL_GetVideoMemoryPool( + IN gckKERNEL Kernel, + IN gcePOOL Pool, + OUT gckVIDMEM * VideoMemory + ); + +/* Map dedicated video memory node. */ +gceSTATUS +gckKERNEL_MapVideoMemory( + IN gckKERNEL Kernel, + IN gctBOOL InUserSpace, + IN gcePOOL Pool, + IN gctPHYS_ADDR Physical, + IN gctUINT32 Offset, + IN gctUINT32 Bytes, + OUT gctPOINTER * Logical + ); + +/* Unmap dedicated video memory. */ +gceSTATUS +gckKERNEL_UnmapVideoMemory( + IN gckKERNEL Kernel, + IN gcePOOL Pool, + IN gctPHYS_ADDR Physical, + IN gctPOINTER Logical, + IN gctUINT32 Pid, + IN gctSIZE_T Bytes + ); + +/* Map memory. */ +gceSTATUS +gckKERNEL_MapMemory( + IN gckKERNEL Kernel, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Logical + ); + +/* Unmap memory. */ +gceSTATUS +gckKERNEL_UnmapMemory( + IN gckKERNEL Kernel, + IN gctPHYS_ADDR Physical, + IN gctSIZE_T Bytes, + IN gctPOINTER Logical, + IN gctUINT32 ProcessID + ); +/* Destroy reserved mem when destroy process*/ +gceSTATUS +gckKERNEL_DestroyProcessReservedUserMap( + IN gckKERNEL Kernel, + IN gctUINT32 Pid + ); + +/* Notification of events. */ +gceSTATUS +gckKERNEL_Notify( + IN gckKERNEL Kernel, + IN gceNOTIFY Notifcation + ); + +/******************************************************************************* +** +** gckKERNEL_Recovery +** +** Try to recover the GPU from a fatal error. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckKERNEL_Recovery( + IN gckKERNEL Kernel + ); + +/* Get access to the user data. */ +gceSTATUS +gckKERNEL_OpenUserData( + IN gckKERNEL Kernel, + IN gctBOOL NeedCopy, + IN gctPOINTER StaticStorage, + IN gctPOINTER UserPointer, + IN gctSIZE_T Size, + OUT gctPOINTER * KernelPointer + ); + +/* Release resources associated with the user data connection. */ +gceSTATUS +gckKERNEL_CloseUserData( + IN gckKERNEL Kernel, + IN gctBOOL NeedCopy, + IN gctBOOL FlushData, + IN gctPOINTER UserPointer, + IN gctSIZE_T Size, + OUT gctPOINTER * KernelPointer + ); + +gceSTATUS +gckDVFS_Construct( + IN gckHARDWARE Hardware, + OUT gckDVFS * Frequency + ); + +gceSTATUS +gckDVFS_Destroy( + IN gckDVFS Dvfs + ); + +gceSTATUS +gckDVFS_Start( + IN gckDVFS Dvfs + ); + +gceSTATUS +gckDVFS_Stop( + IN gckDVFS Dvfs + ); + +/******************************************************************************\ +******************************* gckHARDWARE Object ***************************** +\******************************************************************************/ + +/* Construct a new gckHARDWARE object. */ +gceSTATUS +gckHARDWARE_Construct( + IN gckOS Os, + IN gckDEVICE Device, + IN gceCORE Core, + OUT gckHARDWARE * Hardware + ); + +/* Post hardware resource allocation after gckHARDWARE object constructed. */ +gceSTATUS +gckHARDWARE_PostConstruct( + IN gckHARDWARE Hardware + ); + +/* Pre-destroy hardwre resource before destroying an gckHARDWARE object. */ +gceSTATUS +gckHARDWARE_PreDestroy( + IN gckHARDWARE Hardware + ); + +/* Destroy an gckHARDWARE object. */ +gceSTATUS +gckHARDWARE_Destroy( + IN gckHARDWARE Hardware + ); + +/* Get hardware type. */ +gceSTATUS +gckHARDWARE_GetType( + IN gckHARDWARE Hardware, + OUT gceHARDWARE_TYPE * Type + ); + +/* Query system memory requirements. */ +gceSTATUS +gckHARDWARE_QuerySystemMemory( + IN gckHARDWARE Hardware, + OUT gctSIZE_T * SystemSize, + OUT gctUINT32 * SystemBaseAddress + ); + +/* Build virtual address. */ +gceSTATUS +gckHARDWARE_BuildVirtualAddress( + IN gckHARDWARE Hardware, + IN gctUINT32 Index, + IN gctUINT32 Offset, + OUT gctUINT32 * Address + ); + +/* Query command buffer requirements. */ +gceSTATUS +gckHARDWARE_QueryCommandBuffer( + IN gckHARDWARE Hardware, + IN gceENGINE Engine, + OUT gctUINT32 * Alignment, + OUT gctUINT32 * ReservedHead, + OUT gctUINT32 * ReservedTail + ); + +/* Add a PIPESELECT command in the command queue. */ +gceSTATUS +gckHARDWARE_PipeSelect( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gcePIPE_SELECT Pipe, + IN OUT gctUINT32 * Bytes + ); + +/* Query the available memory. */ +gceSTATUS +gckHARDWARE_QueryMemory( + IN gckHARDWARE Hardware, + OUT gctSIZE_T * InternalSize, + OUT gctUINT32 * InternalBaseAddress, + OUT gctUINT32 * InternalAlignment, + OUT gctSIZE_T * ExternalSize, + OUT gctUINT32 * ExternalBaseAddress, + OUT gctUINT32 * ExternalAlignment, + OUT gctUINT32 * HorizontalTileSize, + OUT gctUINT32 * VerticalTileSize + ); + +/* Query the identity of the hardware. */ +gceSTATUS +gckHARDWARE_QueryChipIdentity( + IN gckHARDWARE Hardware, + OUT gcsHAL_QUERY_CHIP_IDENTITY_PTR Identity + ); + +gceSTATUS +gckHARDWARE_QueryChipOptions( + IN gckHARDWARE Hardware, + OUT gcsHAL_QUERY_CHIP_OPTIONS_PTR Options + ); + + +/* Split a harwdare specific address into API stuff. */ +gceSTATUS +gckHARDWARE_SplitMemory( + IN gckHARDWARE Hardware, + IN gctUINT32 Address, + OUT gcePOOL * Pool, + OUT gctUINT32 * Offset + ); + +/* Update command queue tail pointer. */ +gceSTATUS +gckHARDWARE_UpdateQueueTail( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Offset + ); + +/* Interrupt manager. */ +gceSTATUS +gckHARDWARE_Interrupt( + IN gckHARDWARE Hardware + ); + +gceSTATUS +gckHARDWARE_Notify( + IN gckHARDWARE Hardware + ); + +/* Program MMU. */ +gceSTATUS +gckHARDWARE_SetMMU( + IN gckHARDWARE Hardware, + IN gckMMU Mmu + ); + +/* Flush the MMU. */ +gceSTATUS +gckHARDWARE_FlushMMU( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN gctUINT32 SubsequentBytes, + IN OUT gctUINT32 * Bytes + ); + +gceSTATUS +gckHARDWARE_FlushAsyncMMU( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctUINT32 * Bytes + ); + +gceSTATUS +gckHARDWARE_FlushMcfeMMU( + IN gckHARDWARE Hardware, + IN gctPOINTER Logical, + IN OUT gctUINT32 * Bytes + ); + +/* Get idle register. */ +gceSTATUS +gckHARDWARE_GetIdle( + IN gckHARDWARE Hardware, + IN gctBOOL Wait, + OUT gctUINT32 * Data + ); + +/* Flush the caches. */ +gceSTATUS +gckHARDWARE_Flush( + IN gckHARDWARE Hardware, + IN gceKERNEL_FLUSH Flush, + IN gctPOINTER Logical, + IN OUT gctUINT32 * Bytes + ); + +/* Enable/disable fast clear. */ +gceSTATUS +gckHARDWARE_SetFastClear( + IN gckHARDWARE Hardware, + IN gctINT Enable, + IN gctINT Compression + ); + +gceSTATUS +gckHARDWARE_ReadInterrupt( + IN gckHARDWARE Hardware, + OUT gctUINT32_PTR IDs + ); + +/* +* State timer helper. +*/ +gceSTATUS +gckHARDWARE_StartTimerReset( + IN gckHARDWARE Hardware + ); + +/* Power management. */ +gceSTATUS +gckHARDWARE_SetPowerState( + IN gckHARDWARE Hardware, + IN gceCHIPPOWERSTATE State + ); + +gceSTATUS +gckHARDWARE_QueryPowerState( + IN gckHARDWARE Hardware, + OUT gceCHIPPOWERSTATE* State + ); + +gceSTATUS +gckHARDWARE_EnablePowerManagement( + IN gckHARDWARE Hardware, + IN gctBOOL Enable + ); + +gceSTATUS +gckHARDWARE_SetGpuProfiler( + IN gckHARDWARE Hardware, + IN gctBOOL GpuProfiler + ); + +#if gcdENABLE_FSCALE_VAL_ADJUST +gceSTATUS +gckHARDWARE_SetFscaleValue( + IN gckHARDWARE Hardware, + IN gctUINT32 FscaleValue, + IN gctUINT32 ShaderFscaleValue + ); + +gceSTATUS +gckHARDWARE_GetFscaleValue( + IN gckHARDWARE Hardware, + IN gctUINT * FscaleValue, + IN gctUINT * MinFscaleValue, + IN gctUINT * MaxFscaleValue + ); + +gceSTATUS +gckHARDWARE_SetMinFscaleValue( + IN gckHARDWARE Hardware, + IN gctUINT MinFscaleValue + ); +#endif + +gceSTATUS +gckHARDWARE_InitializeHardware( + IN gckHARDWARE Hardware + ); + +gceSTATUS +gckHARDWARE_Reset( + IN gckHARDWARE Hardware + ); + +/* Check for Hardware features. */ +gceSTATUS +gckHARDWARE_IsFeatureAvailable( + IN gckHARDWARE Hardware, + IN gceFEATURE Feature + ); + +gceSTATUS +gckHARDWARE_DumpMMUException( + IN gckHARDWARE Hardware + ); + +gceSTATUS +gckHARDWARE_DumpGPUState( + IN gckHARDWARE Hardware + ); + +gceSTATUS +gckHARDWARE_InitDVFS( + IN gckHARDWARE Hardware + ); + +gceSTATUS +gckHARDWARE_QueryLoad( + IN gckHARDWARE Hardware, + OUT gctUINT32 * Load + ); + +gceSTATUS +gckHARDWARE_SetDVFSPeroid( + IN gckHARDWARE Hardware, + IN gctUINT32 Frequency + ); + +gceSTATUS +gckHARDWARE_QueryStateTimer( + IN gckHARDWARE Hardware, + OUT gctUINT64_PTR On, + OUT gctUINT64_PTR Off, + OUT gctUINT64_PTR Idle, + OUT gctUINT64_PTR Suspend + ); + +gceSTATUS +gckHARDWARE_Fence( + IN gckHARDWARE Hardware, + IN gceENGINE Engine, + IN gctPOINTER Logical, + IN gctUINT32 FenceAddress, + IN gctUINT64 FenceData, + IN OUT gctUINT32 * Bytes + ); + +/******************************************************************************\ +***************************** gckINTERRUPT Object ****************************** +\******************************************************************************/ + +typedef struct _gckINTERRUPT * gckINTERRUPT; + +typedef gceSTATUS (* gctINTERRUPT_HANDLER)( + IN gckKERNEL Kernel + ); + +gceSTATUS +gckINTERRUPT_Construct( + IN gckKERNEL Kernel, + OUT gckINTERRUPT * Interrupt + ); + +gceSTATUS +gckINTERRUPT_Destroy( + IN gckINTERRUPT Interrupt + ); + +gceSTATUS +gckINTERRUPT_SetHandler( + IN gckINTERRUPT Interrupt, + IN OUT gctINT32_PTR Id, + IN gctINTERRUPT_HANDLER Handler + ); + +gceSTATUS +gckINTERRUPT_Notify( + IN gckINTERRUPT Interrupt, + IN gctBOOL Valid + ); + +/******************************************************************************\ +********************************* gckMMU Object ******************************** +\******************************************************************************/ + +/* Construct a new gckMMU object. */ +gceSTATUS +gckMMU_Construct( + IN gckKERNEL Kernel, + IN gctSIZE_T MmuSize, + OUT gckMMU * Mmu + ); + +/* Destroy an gckMMU object. */ +gceSTATUS +gckMMU_Destroy( + IN gckMMU Mmu + ); + +/* Allocate pages inside the MMU. */ +gceSTATUS +gckMMU_AllocatePages( + IN gckMMU Mmu, + IN gctSIZE_T PageCount, + IN gcePAGE_TYPE PageType, + OUT gctPOINTER * PageTable, + OUT gctUINT32 * Address + ); + +gceSTATUS +gckMMU_AllocatePagesEx( + IN gckMMU Mmu, + IN gctSIZE_T PageCount, + IN gceVIDMEM_TYPE Type, + IN gcePAGE_TYPE PageType, + IN gctBOOL Secure, + OUT gctPOINTER * PageTable, + OUT gctUINT32 * Address + ); + +/* Remove a page table from the MMU. */ +gceSTATUS +gckMMU_FreePages( + IN gckMMU Mmu, + IN gctBOOL Secure, + IN gcePAGE_TYPE PageType, + IN gctUINT32 Address, + IN gctPOINTER PageTable, + IN gctSIZE_T PageCount + ); + +/* Set the MMU page with info. */ +gceSTATUS +gckMMU_SetPage( + IN gckMMU Mmu, + IN gctPHYS_ADDR_T PageAddress, + IN gcePAGE_TYPE PageType, + IN gctBOOL Writable, + IN gctUINT32 *PageEntry + ); + +gceSTATUS +gckMMU_Flush( + IN gckMMU Mmu, + IN gceVIDMEM_TYPE Type + ); + +gceSTATUS +gckMMU_DumpPageTableEntry( + IN gckMMU Mmu, + IN gceAREA_TYPE AreaType, + IN gctUINT32 Address + ); + +gceSTATUS +gckMMU_FillFlatMapping( + IN gckMMU Mmu, + IN gctUINT64 PhysBase, + IN gctSIZE_T Size + ); + +gceSTATUS +gckMMU_IsFlatMapped( + IN gckMMU Mmu, + IN gctUINT64 Physical, + IN gctUINT32 Address, + OUT gctBOOL *In + ); + +gceSTATUS +gckMMU_GetAreaType( + IN gckMMU Mmu, + IN gctUINT32 GpuAddress, + OUT gceAREA_TYPE *AreaType + ); + +gceSTATUS +gckHARDWARE_QueryContextProfile( + IN gckHARDWARE Hardware, + IN gctBOOL Reset, + IN gckCONTEXT Context, + OUT gcsPROFILER_COUNTERS_PART1 * Counters_part1, + OUT gcsPROFILER_COUNTERS_PART2 * Counters_part2 + ); + +gceSTATUS +gckHARDWARE_UpdateContextProfile( + IN gckHARDWARE Hardware, + IN gckCONTEXT Context + ); + +gceSTATUS +gckHARDWARE_InitProfiler( + IN gckHARDWARE Hardware + ); + +gceSTATUS +gckOS_DetectProcessByName( + IN gctCONST_POINTER Name + ); + +void +gckOS_DumpParam( + void + ); + +#ifdef __cplusplus +} +#endif + + +#endif /* __gc_hal_h_ */ + +
diff --git a/hal/kernel/inc/gc_hal_base.h b/hal/kernel/inc/gc_hal_base.h new file mode 100644 index 0000000..80b44a2 --- /dev/null +++ b/hal/kernel/inc/gc_hal_base.h
@@ -0,0 +1,5759 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_base_h_ +#define __gc_hal_base_h_ + +#include "gc_hal_enum.h" +#include "shared/gc_hal_types.h" +#include "gc_hal_debug_zones.h" +#include "shared/gc_hal_base.h" + + +#ifdef __cplusplus +extern "C" { +#endif + +/******************************************************************************\ +****************************** Object Declarations ***************************** +\******************************************************************************/ + +typedef struct _gckOS * gckOS; +typedef struct _gcoHAL * gcoHAL; +typedef struct _gcoOS * gcoOS; +typedef struct _gco2D * gco2D; +typedef struct gcsATOM * gcsATOM_PTR; + +typedef struct _gco3D * gco3D; +typedef struct _gcoCL * gcoCL; +typedef struct _gcoVX * gcoVX; +typedef struct _gcsFAST_FLUSH * gcsFAST_FLUSH_PTR; + +typedef struct _gcoSURF * gcoSURF; +typedef struct _gcsSURF_NODE * gcsSURF_NODE_PTR; +typedef struct _gcsSURF_FORMAT_INFO * gcsSURF_FORMAT_INFO_PTR; +typedef struct _gcsPOINT * gcsPOINT_PTR; +typedef struct _gcsSIZE * gcsSIZE_PTR; +typedef struct _gcsRECT * gcsRECT_PTR; +typedef struct _gcsBOUNDARY * gcsBOUNDARY_PTR; +typedef struct _gcoHARDWARE * gcoHARDWARE; +typedef union _gcuVIDMEM_NODE * gcuVIDMEM_NODE_PTR; +typedef struct _gcsVIDMEM_NODE * gckVIDMEM_NODE; +typedef struct _gcsVIDMEM_BLOCK * gckVIDMEM_BLOCK; + +typedef void * gcoVG; + +typedef struct _gcoFENCE * gcoFENCE; +typedef struct _gcsSYNC_CONTEXT * gcsSYNC_CONTEXT_PTR; + +typedef struct _gcsUSER_MEMORY_DESC * gcsUSER_MEMORY_DESC_PTR; + +/* Immuatable features from database */ +typedef struct _gcsNN_FIXED_FEATURE +{ + gctUINT vipCoreCount; + gctUINT nnCoreCount; /* total nn core count */ + gctUINT nnCoreCountInt8; /* total nn core count supporting int8 */ + gctUINT nnCoreCountInt16; /* total nn core count supporting int16 */ + gctUINT nnCoreCountFloat16; /* total nn core count supporting float16 */ + gctUINT nnCoreCountBFloat16; /* total nn core count supporting Bfloat16 */ + gctUINT nnMadPerCore; + gctUINT nnInputBufferDepth; + gctUINT nnAccumBufferDepth; + gctUINT nnFCNonPrunAccel; + gctUINT nnInImageOffsetBits; + gctUINT tpCoreCount; /* full-function core count */ + gctUINT tpPwlLUTCount; + gctUINT tpPwlLUTSize; + gctUINT vip7Version; + gctUINT vipBrickMode; + gctUINT tpReorderInImageSize; + gctUINT tpliteCoreCount; /* fc-only core count */ + gctUINT nnFP16XYDPX; + gctUINT nnFP16XYDPY; + gctUINT nnFP16ZDP; + gctUINT zrlBits; + gctUINT uscCacheControllers; + gctUINT uscBanks; + gctUINT nnLanesPerOutCycle; + gctUINT maxOTNumber; + gctUINT physicalVipSramWidthInByte; + gctUINT equivalentVipsramWidthInByte; + gctUINT shaderCoreCount; + gctUINT latencyHidingAtFullAxiBw; + gctUINT axiBusWidth; + gctUINT nnMaxKXSize; + gctUINT nnMaxKYSize; + gctUINT nnMaxKZSize; +} gcsNN_FIXED_FEATURE; + +/* Features can be customized from outside */ +typedef struct _gcsNN_CUSTOMIZED_FEATURE +{ + gctUINT vipSRAMSize; + gctUINT axiSRAMSize; + gctFLOAT ddrReadBWLimit; + gctFLOAT ddrWriteBWLimit; + gctFLOAT ddrTotalBWLimit; + gctFLOAT axiSramReadBWLimit; + gctFLOAT axiSramWriteBWLimit; + gctFLOAT axiSramTotalBWLimit; + gctFLOAT axiBusReadBWLimit; + gctFLOAT axiBusWriteBWLimit; + gctFLOAT axiBusTotalBWLimit; + gctUINT vipSWTiling; + gctFLOAT ddrLatency; + gctUINT freqInMHZ; + gctUINT axiClockFreqInMHZ; + gctUINT maxSocOTNumber;/*max SOC outstanding transfer number*/ + gctUINT nnWriteWithoutUSC; + gctUINT depthWiseSupport; + gctUINT vipVectorPrune; + gctUINT ddrKernelBurstSize; +} gcsNN_CUSTOMIZED_FEATURE; + +/* Features are unified (hardcoded) for hardwares */ +typedef struct _gcsNN_UNIFIED_FEATURE +{ + gctUINT nnUSCCacheSize; + gctUINT nnCmdSizeInBytes; + gctUINT tpCmdSizeInBytes; + gctUINT vipCoefDecodePerf; + gctUINT vipCachedReadFromSram; + gctUINT vipImagePartialCache; + gctUINT lanesPerConv; + gctUINT maxTileSize; + gctUINT fullCacheKernelHeadFix : 1; + gctUINT conv1x1HalfPerformance : 1; + gctUINT per3DTileBubbleFix : 1; + gctUINT cacheLineModeDisabled : 1; + gctUINT tpReOrderFix : 1; + gctUINT zdp3NoCompressFix : 1; + gctUINT asyncCopyPerfFix : 1; + gctUINT accurateTileBW : 1; + gctUINT zxdp3KernelReadConflictFix : 1; + gctUINT axiSramSlowedDownByAddr:1; + gctUINT slowNNReqArbitrationFix:1; + gctUINT singlePortAccBuffer : 1; + gctUINT convOutFifoDepthFix : 1; + gctUINT smallBatchEnable : 1; + gctUINT axiSramOnlySWTiling : 1; + gctUINT imageNotPackedInSram : 1; + gctUINT coefDeltaCordOverFlowZRL8BitFix : 1; + gctUINT lowEfficiencyOfIDWriteImgBufFix : 1; + gctUINT xyOffsetLimitationFix : 1; + gctUINT kernelPerCoreLTOneThirdCoefFix : 1; + gctUINT diffConditionForCachelineModePreFix : 1; +} gcsNN_UNIFIED_FEATURE; + +/* Features are derived from above ones */ +typedef struct _gcsNN_DERIVIED_FEATURE +{ + gctUINT nnDPAmount; + gctUINT nnXYDPX; + gctUINT nnXYDPY; + gctUINT nnZDP; + gctFLOAT totalLatency; + gctFLOAT internalLatency; + gctFLOAT ddrReadBWInBytePerCycle; + gctFLOAT ddrWriteBWInBytePerCycle; +} gcsNN_DERIVED_FEATURE; + +/******************************************************************************\ +********************* Share obj lock/unlock macros. **************************** +\******************************************************************************/ +#define gcmLOCK_SHARE_OBJ(Obj) \ +{ \ + if(Obj->sharedLock != gcvNULL)\ + {\ + (gcoOS_AcquireMutex(\ + gcvNULL, Obj->sharedLock, gcvINFINITE));\ + }\ +} + + +#define gcmUNLOCK_SHARE_OBJ(Obj)\ +{\ + if(Obj->sharedLock != gcvNULL)\ + {\ + (gcoOS_ReleaseMutex(gcvNULL, Obj->sharedLock));\ + }\ +} + +typedef struct _gcsSystemInfo +{ + /* memory latency number for SH data fetch, in SH cycle*/ + gctUINT32 memoryLatencySH; +} +gcsSystemInfo; + + +#define gcPLS_INITIALIZER \ +{ \ + gcvNULL, /* gcoOS object. */ \ + gcvNULL, /* gcoHAL object. */ \ + 0, /* internalSize */ \ + 0, /* internalPhysName */ \ + gcvNULL, /* internalLogical */ \ + 0, /* externalSize */ \ + 0, /* externalPhysName */ \ + gcvNULL, /* externalLogical */ \ + 0, /* contiguousSize */ \ + 0, /* contiguousPhysName */ \ + gcvNULL, /* contiguousLogical */ \ + gcvNULL, /* eglDisplayInfo */ \ + gcvNULL, /* eglSurfaceInfo */ \ + gcvSURF_A8R8G8B8,/* eglConfigFormat */ \ + gcvNULL, /* reference */ \ + 0, /* processID */ \ + 0, /* threadID */ \ + gcvFALSE, /* exiting */ \ + gcvFALSE, /* Special flag for NP2 texture. */ \ + gcvFALSE, /* device open. */ \ + gcvNULL, /* destructor */ \ + gcvNULL, /* accessLock */ \ + gcvNULL, /* GL FE compiler lock*/ \ + gcvNULL, /* CL FE compiler lock*/ \ + gcvNULL, /* VX context lock */ \ + gcvPATCH_NOTINIT,/* global patchID */ \ + gcvNULL, /* global fenceID*/ \ + gcvNULL, /* mainThreadHandle */ \ + gcvFALSE, /* memory profile flag */ \ + gcvNULL, /* profileLock; */ \ + 0, /* allocCount; */ \ + 0, /* allocSize; */ \ + 0, /* maxAllocSize; */ \ + 0, /* freeCount; */ \ + 0, /* freeSize; */ \ + 0, /* currentSize; */ \ + 0, /* video_allocCount; */ \ + 0, /* video_allocSize; */ \ + 0, /* video_maxAllocSize; */ \ + 0, /* video_freeCount; */ \ + 0, /* video_freeSize; */ \ + 0, /* video_currentSize; */ \ +} + +/******************************************************************************\ +******************************* Thread local storage ************************* +\******************************************************************************/ + +typedef struct _gcsDRIVER_TLS * gcsDRIVER_TLS_PTR; + +typedef struct _gcsDRIVER_TLS +{ + void (* destructor)(gcsDRIVER_TLS_PTR Tls); +} +gcsDRIVER_TLS; + +typedef struct _gcsTLS * gcsTLS_PTR; + +typedef struct _gcsTLS +{ + gceHARDWARE_TYPE currentType; + gceHARDWARE_TYPE targetType; + + /* To which core device control is called, + * it is index in a hardware type. + */ + gctUINT32 currentCoreIndex; + + /* Current 3D hardwre of this thread */ + gcoHARDWARE currentHardware; + + /* Default 3D hardware of this thread */ + gcoHARDWARE defaultHardware; + + /* Only for separated 3D and 2D */ + gcoHARDWARE hardware2D; +#if gcdENABLE_3D + gco3D engine3D; +#endif + gcoVX engineVX; + + gctBOOL copied; + + /* libGAL.so handle */ + gctHANDLE handle; + + /* If true, do not releas 2d engine and hardware in hal layer */ + gctBOOL release2DUpper; + + /* Driver tls. */ + gcsDRIVER_TLS_PTR driverTLS[gcvTLS_KEY_COUNT]; +} +gcsTLS; + +typedef struct _gcsSURF_VIEW +{ + gcoSURF surf; + gctUINT firstSlice; + gctUINT numSlices; +}gcsSURF_VIEW; + +/* gcsHAL_Limits*/ +typedef struct _gcsHAL_LIMITS +{ + /* chip info */ + gceCHIPMODEL chipModel; + gctUINT32 chipRevision; + gctUINT32 featureCount; + gctUINT32 *chipFeatures; + + /* target caps */ + gctUINT32 maxWidth; + gctUINT32 maxHeight; + gctUINT32 multiTargetCount; + gctUINT32 maxSamples; + +}gcsHAL_LIMITS; + + +typedef struct _gcsHAL_CHIPIDENTITY +{ + gceCHIPMODEL chipModel; + gctUINT32 chipRevision; + gctUINT32 productID; + gctUINT32 customerID; + gctUINT32 ecoID; + gceCHIP_FLAG chipFlags; + gctUINT64 platformFlagBits; +} +gcsHAL_CHIPIDENTITY; + +/******************************************************************************\ +********************************* gcoHAL Object ********************************* +\******************************************************************************/ + +/* Construct a new gcoHAL object. */ +gceSTATUS +gcoHAL_ConstructEx( + IN gctPOINTER Context, + IN gcoOS Os, + OUT gcoHAL * Hal + ); + +/* Destroy an gcoHAL object. */ +gceSTATUS +gcoHAL_DestroyEx( + IN gcoHAL Hal + ); + +/* Empty function for compatibility. */ +gceSTATUS +gcoHAL_Construct( + IN gctPOINTER Context, + IN gcoOS Os, + OUT gcoHAL * Hal + ); + +/* Empty function for compatibility. */ +gceSTATUS +gcoHAL_Destroy( + IN gcoHAL Hal + ); + +/* Get HAL options */ +gceSTATUS +gcoHAL_GetOption( + IN gcoHAL Hal, + IN gceOPTION Option + ); + +gceSTATUS +gcoHAL_FrameInfoOps( + IN gcoHAL Hal, + IN gceFRAMEINFO FrameInfo, + IN gceFRAMEINFO_OP Op, + IN OUT gctUINT * Val + ); + +/* Set HAL options */ +gceSTATUS +gcoHAL_SetOption( + IN gcoHAL Hal, + IN gceOPTION Option, + IN gctBOOL Value + ); + +gceSTATUS +gcoHAL_GetHardware( + IN gcoHAL Hal, + OUT gcoHARDWARE* Hw + ); + + +#if gcdENABLE_3D +gceSTATUS +gcoHAL_GetSpecialHintData( + IN gcoHAL Hal, + OUT gctINT * Hint + ); +/* +** Deprecated(Don't use it), keep it here for external library(libgcu.so) +*/ +gceSTATUS +gcoHAL_Get3DEngine( + IN gcoHAL Hal, + OUT gco3D * Engine + ); +#endif /* gcdENABLE_3D */ + + +gceSTATUS +gcoHAL_GetProductName( + IN gcoHAL Hal, + OUT gctSTRING *ProductName, + OUT gctUINT *PID + ); + +gceSTATUS +gcoHAL_SetFscaleValue( + IN gcoHAL Hal, + IN gctUINT CoreIndex, + IN gctUINT FscaleValue, + IN gctUINT ShaderFscaleValue + ); + +gceSTATUS +gcoHAL_GetFscaleValue( + OUT gctUINT * FscaleValue, + OUT gctUINT * MinFscaleValue, + OUT gctUINT * MaxFscaleValue + ); + +gceSTATUS +gcoHAL_SetBltNP2Texture( + gctBOOL enable + ); + +gceSTATUS +gcoHAL_ExportVideoMemory( + IN gctUINT32 Handle, + IN gctUINT32 Flags, + OUT gctINT32 * FD + ); + +gceSTATUS +gcoHAL_NameVideoMemory( + IN gctUINT32 Handle, + OUT gctUINT32 * Name + ); + +gceSTATUS +gcoHAL_ImportVideoMemory( + IN gctUINT32 Name, + OUT gctUINT32 * Handle + ); + +gceSTATUS +gcoHAL_GetVideoMemoryFd( + IN gctUINT32 Handle, + OUT gctINT * Fd + ); + +/* Verify whether the specified feature is available in hardware. */ +gceSTATUS +gcoHAL_IsFeatureAvailable( + IN gcoHAL Hal, + IN gceFEATURE Feature + ); + +gceSTATUS +gcoHAL_IsFeatureAvailable1( + IN gcoHAL Hal, + IN gceFEATURE Feature + ); + +/* Query the identity of the hardware. */ +gceSTATUS +gcoHAL_QueryChipIdentity( + IN gcoHAL Hal, + OUT gceCHIPMODEL* ChipModel, + OUT gctUINT32* ChipRevision, + OUT gctUINT32* ChipFeatures, + OUT gctUINT32* ChipMinorFeatures + ); + +gceSTATUS gcoHAL_QueryChipIdentityEx( + IN gcoHAL Hal, + IN gctUINT32 SizeOfParam, + OUT gcsHAL_CHIPIDENTITY *ChipIdentity + ); + + +gceSTATUS +gcoHAL_QuerySuperTileMode( + OUT gctUINT32_PTR SuperTileMode + ); + +gceSTATUS +gcoHAL_QueryChipAxiBusWidth( + OUT gctBOOL * AXI128Bits + ); + +gceSTATUS +gcoHAL_QueryMultiGPUAffinityConfig( + IN gceHARDWARE_TYPE Type, + OUT gceMULTI_GPU_MODE *Mode, + OUT gctUINT32_PTR CoreIndex + ); + +gceSTATUS +gcoHAL_QuerySRAM( + IN gcoHAL Hal, + IN gcePOOL Type, + OUT gctUINT32 *Size, + OUT gctUINT32 *GPUVirtAddr, + OUT gctPHYS_ADDR_T *GPUPhysAddr, + OUT gctUINT32 *GPUPhysName, + OUT gctPHYS_ADDR_T *CPUPhysAddr + ); + +#ifdef LINUX +gctINT32 +gcoOS_EndRecordAllocation(void); +void +gcoOS_RecordAllocation(void); +void +gcoOS_AddRecordAllocation(gctSIZE_T Size); +#endif + +/* Query the amount of video memory. */ +gceSTATUS +gcoHAL_QueryVideoMemory( + IN gcoHAL Hal, + OUT gctUINT32 * InternalPhysName, + OUT gctSIZE_T * InternalSize, + OUT gctUINT32 * ExternalPhysName, + OUT gctSIZE_T * ExternalSize, + OUT gctUINT32 * ContiguousPhysName, + OUT gctSIZE_T * ContiguousSize + ); + +/* Map video memory. */ +gceSTATUS +gcoHAL_MapMemory( + IN gcoHAL Hal, + IN gctUINT32 PhysName, + IN gctSIZE_T NumberOfBytes, + OUT gctPOINTER * Logical + ); + +/* Unmap video memory. */ +gceSTATUS +gcoHAL_UnmapMemory( + IN gcoHAL Hal, + IN gctUINT32 PhysName, + IN gctSIZE_T NumberOfBytes, + IN gctPOINTER Logical + ); + +/* Schedule an unmap of a buffer mapped through its physical address. */ +gceSTATUS +gcoHAL_ScheduleUnmapMemory( + IN gcoHAL Hal, + IN gctUINT32 PhysName, + IN gctSIZE_T NumberOfBytes, + IN gctPOINTER Logical + ); + +/* Allocate video memory. */ +gceSTATUS +gcoOS_AllocateVideoMemory( + IN gcoOS Os, + IN gctBOOL InUserSpace, + IN gctBOOL InCacheable, + IN OUT gctSIZE_T * Bytes, + OUT gctUINT32 * Physical, + OUT gctPOINTER * Logical, + OUT gctPOINTER * Handle + ); + +/* Free video memory. */ +gceSTATUS +gcoOS_FreeVideoMemory( + IN gcoOS Os, + IN gctPOINTER Handle + ); + +/* Lock video memory. */ +gceSTATUS +gcoOS_LockVideoMemory( + IN gcoOS Os, + IN gctPOINTER Handle, + IN gctBOOL InUserSpace, + IN gctBOOL InCacheable, + OUT gctUINT32 * Address, + OUT gctPOINTER * Logical + ); + +/* Commit the current command buffer. */ +gceSTATUS +gcoHAL_Commit( + IN gcoHAL Hal, + IN gctBOOL Stall + ); + +#if gcdENABLE_3D +/* Sencd fence command. */ +gceSTATUS +gcoHAL_SendFence( + IN gcoHAL Hal + ); +#endif /* gcdENABLE_3D */ + +/* Query the tile capabilities. */ +gceSTATUS +gcoHAL_QueryTiled( + IN gcoHAL Hal, + OUT gctINT32 * TileWidth2D, + OUT gctINT32 * TileHeight2D, + OUT gctINT32 * TileWidth3D, + OUT gctINT32 * TileHeight3D + ); + +gceSTATUS +gcoHAL_Compact( + IN gcoHAL Hal + ); + +#if VIVANTE_PROFILER_SYSTEM_MEMORY +gceSTATUS +gcoHAL_ProfileStart( + IN gcoHAL Hal + ); + +gceSTATUS +gcoHAL_ProfileEnd( + IN gcoHAL Hal, + IN gctCONST_STRING Title + ); +#endif + +/* Power Management */ +gceSTATUS +gcoHAL_SetPowerManagementState( + IN gcoHAL Hal, + IN gceCHIPPOWERSTATE State + ); + +gceSTATUS +gcoHAL_QueryPowerManagementState( + IN gcoHAL Hal, + OUT gceCHIPPOWERSTATE *State + ); + +/* Set the filter type for filter blit. */ +gceSTATUS +gcoHAL_SetFilterType( + IN gcoHAL Hal, + IN gceFILTER_TYPE FilterType + ); + +/* Call the kernel HAL layer. */ +gceSTATUS +gcoHAL_Call( + IN gcoHAL Hal, + IN OUT gcsHAL_INTERFACE_PTR Interface + ); + +/* Schedule an event. */ +gceSTATUS +gcoHAL_ScheduleEvent( + IN gcoHAL Hal, + IN OUT gcsHAL_INTERFACE_PTR Interface + ); + +/* Request a start/stop timestamp. */ +gceSTATUS +gcoHAL_SetTimer( + IN gcoHAL Hal, + IN gctUINT32 Index, + IN gctBOOL Start + ); + +/* Get Time delta from a Timer in microseconds. */ +gceSTATUS +gcoHAL_GetTimerTime( + IN gcoHAL Hal, + IN gctUINT32 Timer, + OUT gctINT32_PTR TimeDelta + ); + +/* set timeout value. */ +gceSTATUS +gcoHAL_SetTimeOut( + IN gcoHAL Hal, + IN gctUINT32 timeOut + ); + +gceSTATUS +gcoHAL_SetHardwareType( + IN gcoHAL Hal, + IN gceHARDWARE_TYPE HardwardType + ); + +gceSTATUS +gcoHAL_GetHardwareType( + IN gcoHAL Hal, + OUT gceHARDWARE_TYPE * HardwardType + ); + +gceSTATUS +gcoHAL_QueryChipCount( + IN gcoHAL Hal, + OUT gctINT32 * Count + ); + +gceSTATUS +gcoHAL_Query3DCoreCount( + IN gcoHAL Hal, + OUT gctUINT32 *Count + ); + +gceSTATUS +gcoHAL_QueryCluster( + IN gcoHAL Hal, + OUT gctINT32 *ClusterMinID, + OUT gctINT32 *ClusterMaxID, + OUT gctUINT32 *ClusterCount, + OUT gctUINT32 *ClusterIDWidth + ); + +gceSTATUS +gcoHAL_QueryCoreCount( + IN gcoHAL Hal, + IN gceHARDWARE_TYPE Type, + OUT gctUINT *Count, + OUT gctUINT_PTR ChipIDs + ); + +gceSTATUS +gcoHAL_QuerySeparated2D( + IN gcoHAL Hal + ); + +gceSTATUS +gcoHAL_QueryHybrid2D( + IN gcoHAL Hal + ); + +gceSTATUS +gcoHAL_Is3DAvailable( + IN gcoHAL Hal + ); + +/* Get pointer to gcoVG object. */ +gceSTATUS +gcoHAL_GetVGEngine( + IN gcoHAL Hal, + OUT gcoVG * Engine + ); + +gceSTATUS +gcoHAL_QueryChipLimits( + IN gcoHAL Hal, + IN gctINT32 Chip, + OUT gcsHAL_LIMITS *Limits); + +gceSTATUS +gcoHAL_QueryChipFeature( + IN gcoHAL Hal, + IN gctINT32 Chip, + IN gceFEATURE Feature); + +gceSTATUS +gcoHAL_SetCoreIndex( + IN gcoHAL Hal, + IN gctUINT32 Core + ); + +gceSTATUS +gcoHAL_GetCurrentCoreIndex( + IN gcoHAL Hal, + OUT gctUINT32 *Core + ); + +gceSTATUS +gcoHAL_ConvertCoreIndexGlobal( + IN gcoHAL Hal, + IN gceHARDWARE_TYPE Type, + IN gctUINT32 CoreCount, + IN gctUINT32 *LocalCoreIndexs, + OUT gctUINT32 *GlobalCoreIndexs + ); + +gceSTATUS +gcoHAL_ConvertCoreIndexLocal( + IN gcoHAL Hal, + IN gceHARDWARE_TYPE Type, + IN gctUINT32 CoreCount, + IN gctUINT32 *GlobalCoreIndexs, + OUT gctUINT32 *LocalCoreIndexs + ); + +gceSTATUS +gcoHAL_SelectChannel( + IN gcoHAL Hal, + IN gctBOOL Priority, + IN gctUINT32 ChannelId + ); + +gceSTATUS +gcoHAL_MCFESemaphore( + IN gctUINT32 SemaHandle, + IN gctBOOL SendSema + ); + +gceSTATUS +gcoHAL_AllocateMCFESemaphore( + OUT gctUINT32 * SemaHandle + ); + +gceSTATUS +gcoHAL_FreeMCFESemaphore( + IN gctUINT32 SemaHandle + ); + +/*----------------------------------------------------------------------------*/ +/*----- Shared Buffer --------------------------------------------------------*/ + +/* Create shared buffer. */ +gceSTATUS +gcoHAL_CreateShBuffer( + IN gctUINT32 Size, + OUT gctSHBUF * ShBuf + ); + +/* Destroy shared buffer. */ +gceSTATUS +gcoHAL_DestroyShBuffer( + IN gctSHBUF ShBuf + ); + +/* Map shared buffer to current process. */ +gceSTATUS +gcoHAL_MapShBuffer( + IN gctSHBUF ShBuf + ); + +/* Write user data to shared buffer. */ +gceSTATUS +gcoHAL_WriteShBuffer( + IN gctSHBUF ShBuf, + IN gctCONST_POINTER Data, + IN gctUINT32 ByteCount + ); + +/* Read user data from shared buffer. */ +gceSTATUS +gcoHAL_ReadShBuffer( + IN gctSHBUF ShBuf, + IN gctPOINTER Data, + IN gctUINT32 BytesCount, + OUT gctUINT32 * BytesRead + ); + +/* Config power management to be enabled or disabled. */ +gceSTATUS +gcoHAL_ConfigPowerManagement( + IN gctBOOL Enable + ); + +gceSTATUS +gcoHAL_AllocateVideoMemory( + IN gctUINT Alignment, + IN gceVIDMEM_TYPE Type, + IN gctUINT32 Flag, + IN gcePOOL Pool, + IN OUT gctSIZE_T * Bytes, + OUT gctUINT32_PTR Node + ); + +gceSTATUS +gcoHAL_LockVideoMemory( + IN gctUINT32 Node, + IN gctBOOL Cacheable, + IN gceENGINE engine, + OUT gctUINT32 * Address, + OUT gctPOINTER * Logical + ); + +gceSTATUS +gcoHAL_UnlockVideoMemory( + IN gctUINT32 Node, + IN gceVIDMEM_TYPE Type, + IN gceENGINE engine + ); + +gceSTATUS +gcoHAL_UnlockVideoMemoryEX( + IN gctUINT32 Node, + IN gceVIDMEM_TYPE Type, + IN gceENGINE Engine, + IN gctBOOL Sync + ); + +gceSTATUS +gcoHAL_ReleaseVideoMemory( + IN gctUINT32 Node + ); + +#if gcdENABLE_3D +/* Query the target capabilities. */ +gceSTATUS +gcoHAL_QueryTargetCaps( + IN gcoHAL Hal, + OUT gctUINT * MaxWidth, + OUT gctUINT * MaxHeight, + OUT gctUINT * MultiTargetCount, + OUT gctUINT * MaxSamples + ); +#endif + +gceSTATUS +gcoHAL_WrapUserMemory( + IN gcsUSER_MEMORY_DESC_PTR UserMemoryDesc, + IN gceVIDMEM_TYPE Type, + OUT gctUINT32_PTR Node + ); + +gceSTATUS +gcoHAL_QueryResetTimeStamp( + OUT gctUINT64_PTR ResetTimeStamp, + OUT gctUINT64_PTR ContextID + ); + +gceSTATUS +gcoHAL_WaitFence( + IN gctUINT32 Handle, + IN gctUINT32 TimeOut + ); + + +gceSTATUS +gcoHAL_ScheduleSignal( + IN gctSIGNAL Signal, + IN gctSIGNAL AuxSignal, + IN gctINT ProcessID, + IN gceKERNEL_WHERE FromWhere + ); + +gceSTATUS +gcoHAL_GetGraphicBufferFd( + IN gctUINT32 Node[3], + IN gctSHBUF ShBuf, + IN gctSIGNAL Signal, + OUT gctINT32 * Fd + ); + +gceSTATUS +gcoHAL_AlignToTile( + IN OUT gctUINT32 * Width, + IN OUT gctUINT32 * Height, + IN gceSURF_TYPE Type, + IN gceSURF_FORMAT Format + ); + +/******************************************************************************\ +********************************** gcoOS Object ********************************* +\******************************************************************************/ +/* Lock PLS access */ +gceSTATUS +gcoOS_LockPLS( + void + ); + +/* Unlock PLS access */ +gceSTATUS +gcoOS_UnLockPLS( + void + ); + +/* Get PLS value for given key */ +gctPOINTER +gcoOS_GetPLSValue( + IN gcePLS_VALUE key + ); + +/* Set PLS value of a given key */ +void +gcoOS_SetPLSValue( + IN gcePLS_VALUE key, + OUT gctPOINTER value + ); + +/* Lock GL FE compiler access */ +gceSTATUS +gcoOS_LockGLFECompiler( + void + ); + +/* Unlock GL FE compiler access */ +gceSTATUS +gcoOS_UnLockGLFECompiler( + void + ); + +/* Lock CL FE compiler access */ +gceSTATUS +gcoOS_LockCLFECompiler( + void + ); + +/* Unlock CL FE compiler access */ +gceSTATUS +gcoOS_UnLockCLFECompiler( + void + ); + +gceSTATUS +gcoOS_GetTLS( + OUT gcsTLS_PTR * TLS + ); + +/* Copy the TLS from a source thread. */ +gceSTATUS +gcoOS_CopyTLS( + IN gcsTLS_PTR Source + ); + +/* Query the thread local storage. */ +gceSTATUS +gcoOS_QueryTLS( + OUT gcsTLS_PTR * TLS + ); + +/* Get access to driver tls. */ +gceSTATUS +gcoOS_GetDriverTLS( + IN gceTLS_KEY Key, + OUT gcsDRIVER_TLS_PTR * TLS + ); + +/* + * Set driver tls. + * May cause memory leak if 'destructor' not set. + */ +gceSTATUS +gcoOS_SetDriverTLS( + IN gceTLS_KEY Key, + IN gcsDRIVER_TLS * TLS + ); + +/* Destroy the objects associated with the current thread. */ +void +gcoOS_FreeThreadData( + void + ); + +/* Empty function for compatibility. */ +gceSTATUS +gcoOS_Construct( + IN gctPOINTER Context, + OUT gcoOS * Os + ); + +/* Empty function for compatibility. */ +gceSTATUS +gcoOS_Destroy( + IN gcoOS Os + ); + +/* Deprecated API: please use gcoHAL_GetBaseAddr() instead. +** This API was kept only for legacy BSP usage. +** +** Get the base address for the physical memory. +*/ +gceSTATUS +gcoOS_GetBaseAddress( + IN gcoOS Os, + OUT gctUINT32_PTR BaseAddress + ); + +/* Allocate memory from the heap. */ +gceSTATUS +gcoOS_Allocate( + IN gcoOS Os, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Memory + ); + +/* Get allocated memory size. */ +gceSTATUS +gcoOS_GetMemorySize( + IN gcoOS Os, + IN gctPOINTER Memory, + OUT gctSIZE_T_PTR MemorySize + ); + +/* Free allocated memory. */ +gceSTATUS +gcoOS_Free( + IN gcoOS Os, + IN gctPOINTER Memory + ); + +/* Allocate memory. */ +gceSTATUS +gcoOS_AllocateSharedMemory( + IN gcoOS Os, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Memory + ); + +/* Free memory. */ +gceSTATUS +gcoOS_FreeSharedMemory( + IN gcoOS Os, + IN gctPOINTER Memory + ); + +/* Allocate memory. */ +gceSTATUS +gcoOS_AllocateMemory( + IN gcoOS Os, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Memory + ); + +/* Free memory. */ +gceSTATUS +gcoOS_FreeMemory( + IN gcoOS Os, + IN gctPOINTER Memory + ); + +/* Device I/O Control call to the kernel HAL layer. */ +gceSTATUS +gcoOS_DeviceControl( + IN gcoOS Os, + IN gctUINT32 IoControlCode, + IN gctPOINTER InputBuffer, + IN gctSIZE_T InputBufferSize, + IN gctPOINTER OutputBuffer, + IN gctSIZE_T OutputBufferSize + ); + +#define gcmOS_SAFE_FREE(os, mem) \ + gcoOS_Free(os, mem); \ + mem = gcvNULL + +#define gcmOS_SAFE_FREE_SHARED_MEMORY(os, mem) \ + gcoOS_FreeSharedMemory(os, mem); \ + mem = gcvNULL + +#define gcmkOS_SAFE_FREE(os, mem) \ + gckOS_Free(os, mem); \ + mem = gcvNULL + +#define gcdMAX_PATH 512 + +/* Open a file. */ +gceSTATUS +gcoOS_Open( + IN gcoOS Os, + IN gctCONST_STRING FileName, + IN gceFILE_MODE Mode, + OUT gctFILE * File + ); + +/* Close a file. */ +gceSTATUS +gcoOS_Close( + IN gcoOS Os, + IN gctFILE File + ); + +/* Remove a file. */ +gceSTATUS +gcoOS_Remove( + IN gcoOS Os, + IN gctCONST_STRING FileName + ); + +/* Read data from a file. */ +gceSTATUS +gcoOS_Read( + IN gcoOS Os, + IN gctFILE File, + IN gctSIZE_T ByteCount, + IN gctPOINTER Data, + OUT gctSIZE_T * ByteRead + ); + +/* Write data to a file. */ +gceSTATUS +gcoOS_Write( + IN gcoOS Os, + IN gctFILE File, + IN gctSIZE_T ByteCount, + IN gctCONST_POINTER Data + ); + +/* Flush data to a file. */ +gceSTATUS +gcoOS_Flush( + IN gcoOS Os, + IN gctFILE File + ); + +/* Close a file descriptor. */ +gceSTATUS +gcoOS_CloseFD( + IN gcoOS Os, + IN gctINT FD + ); + +/* Scan a file. */ +gceSTATUS +gcoOS_FscanfI( + IN gcoOS Os, + IN gctFILE File, + IN gctCONST_STRING Format, + OUT gctUINT *result + ); + +/* Dup file descriptor to another. */ +gceSTATUS +gcoOS_DupFD( + IN gcoOS Os, + IN gctINT FD, + OUT gctINT * FD2 + ); + +/* Lock a file. */ +gceSTATUS +gcoOS_LockFile( + IN gcoOS Os, + IN gctFILE File, + IN gctBOOL Shared, + IN gctBOOL Block + ); + +/* Unlock a file. */ +gceSTATUS +gcoOS_UnlockFile( + IN gcoOS Os, + IN gctFILE File + ); + +/* Create an endpoint for communication. */ +gceSTATUS +gcoOS_Socket( + IN gcoOS Os, + IN gctINT Domain, + IN gctINT Type, + IN gctINT Protocol, + OUT gctINT *SockFd + ); + +/* Close a socket. */ +gceSTATUS +gcoOS_CloseSocket( + IN gcoOS Os, + IN gctINT SockFd + ); + +/* Initiate a connection on a socket. */ +gceSTATUS +gcoOS_Connect( + IN gcoOS Os, + IN gctINT SockFd, + IN gctCONST_POINTER HostName, + IN gctUINT Port); + +/* Shut down part of connection on a socket. */ +gceSTATUS +gcoOS_Shutdown( + IN gcoOS Os, + IN gctINT SockFd, + IN gctINT How + ); + +/* Send a message on a socket. */ +gceSTATUS +gcoOS_Send( + IN gcoOS Os, + IN gctINT SockFd, + IN gctSIZE_T ByteCount, + IN gctCONST_POINTER Data, + IN gctINT Flags + ); + +/* Initiate a connection on a socket. */ +gceSTATUS +gcoOS_WaitForSend( + IN gcoOS Os, + IN gctINT SockFd, + IN gctINT Seconds, + IN gctINT MicroSeconds); + +/* Get environment variable value. */ +gceSTATUS +gcoOS_GetEnv( + IN gcoOS Os, + IN gctCONST_STRING VarName, + OUT gctSTRING * Value + ); + +/* Set environment variable value. */ +gceSTATUS +gcoOS_SetEnv( + IN gcoOS Os, + IN gctCONST_STRING VarName, + IN gctSTRING Value + ); + +/* Get current working directory. */ +gceSTATUS +gcoOS_GetCwd( + IN gcoOS Os, + IN gctINT SizeInBytes, + OUT gctSTRING Buffer + ); + +/* Get file status info. */ +gceSTATUS +gcoOS_Stat( + IN gcoOS Os, + IN gctCONST_STRING FileName, + OUT gctPOINTER Buffer + ); + +/* Set the current position of a file. */ +gceSTATUS +gcoOS_Seek( + IN gcoOS Os, + IN gctFILE File, + IN gctUINT32 Offset, + IN gceFILE_WHENCE Whence + ); + +/* Set the current position of a file. */ +gceSTATUS +gcoOS_SetPos( + IN gcoOS Os, + IN gctFILE File, + IN gctUINT32 Position + ); + +/* Get the current position of a file. */ +gceSTATUS +gcoOS_GetPos( + IN gcoOS Os, + IN gctFILE File, + OUT gctUINT32 * Position + ); + +/* Same as strstr. */ +gceSTATUS +gcoOS_StrStr( + IN gctCONST_STRING String, + IN gctCONST_STRING SubString, + OUT gctSTRING * Output + ); + +/* Find the last occurance of a character inside a string. */ +gceSTATUS +gcoOS_StrFindReverse( + IN gctCONST_STRING String, + IN gctINT8 Character, + OUT gctSTRING * Output + ); + +gceSTATUS +gcoOS_StrDup( + IN gcoOS Os, + IN gctCONST_STRING String, + OUT gctSTRING * Target + ); + +/* Copy a string. */ +gceSTATUS +gcoOS_StrCopySafe( + IN gctSTRING Destination, + IN gctSIZE_T DestinationSize, + IN gctCONST_STRING Source + ); + +/* Append a string. */ +gceSTATUS +gcoOS_StrCatSafe( + IN gctSTRING Destination, + IN gctSIZE_T DestinationSize, + IN gctCONST_STRING Source + ); + +/* Compare two strings. */ +gceSTATUS +gcoOS_StrCmp( + IN gctCONST_STRING String1, + IN gctCONST_STRING String2 + ); + +/* Compare characters of two strings. */ +gceSTATUS +gcoOS_StrNCmp( + IN gctCONST_STRING String1, + IN gctCONST_STRING String2, + IN gctSIZE_T Count + ); + +/* Convert string to float. */ +gceSTATUS +gcoOS_StrToFloat( + IN gctCONST_STRING String, + OUT gctFLOAT * Float + ); + +/* Convert hex string to integer. */ +gceSTATUS gcoOS_HexStrToInt( + IN gctCONST_STRING String, + OUT gctINT * Int + ); + +/* Convert hex string to float. */ +gceSTATUS +gcoOS_HexStrToFloat( + IN gctCONST_STRING String, + OUT gctFLOAT * Float + ); + +/* Convert string to integer. */ +gceSTATUS +gcoOS_StrToInt( + IN gctCONST_STRING String, + OUT gctINT * Int + ); + +gceSTATUS +gcoOS_MemCmp( + IN gctCONST_POINTER Memory1, + IN gctCONST_POINTER Memory2, + IN gctSIZE_T Bytes + ); + +gceSTATUS +gcoOS_PrintStrSafe( + OUT gctSTRING String, + IN gctSIZE_T StringSize, + IN OUT gctUINT * Offset, + IN gctCONST_STRING Format, + ... + ); + +gceSTATUS +gcoOS_LoadLibrary( + IN gcoOS Os, + IN gctCONST_STRING Library, + OUT gctHANDLE * Handle + ); + +gceSTATUS +gcoOS_FreeLibrary( + IN gcoOS Os, + IN gctHANDLE Handle + ); + +gceSTATUS +gcoOS_GetProcAddress( + IN gcoOS Os, + IN gctHANDLE Handle, + IN gctCONST_STRING Name, + OUT gctPOINTER * Function + ); + +gceSTATUS +gcoOS_Compact( + IN gcoOS Os + ); + +gceSTATUS +gcoOS_AddSignalHandler ( + IN gceSignalHandlerType SignalHandlerType + ); + +#if VIVANTE_PROFILER_SYSTEM_MEMORY +gceSTATUS +gcoOS_ProfileStart( + IN gcoOS Os + ); + +gceSTATUS +gcoOS_ProfileEnd( + IN gcoOS Os, + IN gctCONST_STRING Title + ); + +gceSTATUS +gcoOS_SetProfileSetting( + IN gcoOS Os, + IN gctBOOL Enable, + IN gctCONST_STRING FileName + ); +#endif + +/* Get the amount of physical system memory */ +gceSTATUS +gcoOS_GetPhysicalSystemMemorySize( + OUT gctSIZE_T * PhysicalSystemMemorySize + ); + +/* Query the video memory. */ +gceSTATUS +gcoOS_QueryVideoMemory( + IN gcoOS Os, + OUT gctUINT32 * InternalPhysName, + OUT gctSIZE_T * InternalSize, + OUT gctUINT32 * ExternalPhysName, + OUT gctSIZE_T * ExternalSize, + OUT gctUINT32 * ContiguousPhysName, + OUT gctSIZE_T * ContiguousSize + ); + +gceSTATUS +gcoOS_QueryCurrentProcessName( + OUT gctSTRING Name, + IN gctSIZE_T Size + ); + + +/*----------------------------------------------------------------------------*/ +/*----- Atoms ----------------------------------------------------------------*/ + +/* Construct an atom. */ +gceSTATUS +gcoOS_AtomConstruct( + IN gcoOS Os, + OUT gcsATOM_PTR * Atom + ); + +/* Destroy an atom. */ +gceSTATUS +gcoOS_AtomDestroy( + IN gcoOS Os, + IN gcsATOM_PTR Atom + ); + +/* Get the 32-bit value protected by an atom. */ +gceSTATUS +gcoOS_AtomGet( + IN gcoOS Os, + IN gcsATOM_PTR Atom, + OUT gctINT32_PTR Value + ); + +/* Set the 32-bit value protected by an atom. */ +gceSTATUS +gcoOS_AtomSet( + IN gcoOS Os, + IN gcsATOM_PTR Atom, + IN gctINT32 Value + ); + +/* Increment an atom. */ +gceSTATUS +gcoOS_AtomIncrement( + IN gcoOS Os, + IN gcsATOM_PTR Atom, + OUT gctINT32_PTR OldValue + ); + +/* Decrement an atom. */ +gceSTATUS +gcoOS_AtomDecrement( + IN gcoOS Os, + IN gcsATOM_PTR Atom, + OUT gctINT32_PTR OldValue + ); + +gctHANDLE +gcoOS_GetCurrentProcessID( + void + ); + +gctHANDLE +gcoOS_GetCurrentThreadID( + void + ); + +/*----------------------------------------------------------------------------*/ +/*----- Time -----------------------------------------------------------------*/ + +/* Get the number of milliseconds since the system started. */ +gctUINT32 +gcoOS_GetTicks( + void + ); + +/* Get time in microseconds. */ +gceSTATUS +gcoOS_GetTime( + gctUINT64_PTR Time + ); + +/* Get CPU usage in microseconds. */ +gceSTATUS +gcoOS_GetCPUTime( + gctUINT64_PTR CPUTime + ); + +/* Get memory usage. */ +gceSTATUS +gcoOS_GetMemoryUsage( + gctUINT32_PTR MaxRSS, + gctUINT32_PTR IxRSS, + gctUINT32_PTR IdRSS, + gctUINT32_PTR IsRSS + ); + +/* Delay a number of microseconds. */ +gceSTATUS +gcoOS_Delay( + IN gcoOS Os, + IN gctUINT32 Delay + ); + +/*----------------------------------------------------------------------------*/ +/*----- Threads --------------------------------------------------------------*/ + +#ifdef _WIN32 +/* Cannot include windows.h here becuase "near" and "far" + * which are used in gcsDEPTH_INFO, are defined to nothing in WinDef.h. + * So, use the real value of DWORD and WINAPI, instead. + * DWORD is unsigned long, and WINAPI is __stdcall. + * If these two are change in WinDef.h, the following two typdefs + * need to be changed, too. + */ +typedef unsigned long gctTHREAD_RETURN; +typedef unsigned long (__stdcall * gcTHREAD_ROUTINE)(void * Argument); +#else +typedef void * gctTHREAD_RETURN; +typedef void * (* gcTHREAD_ROUTINE)(void *); +#endif + +/* Create a new thread. */ +gceSTATUS +gcoOS_CreateThread( + IN gcoOS Os, + IN gcTHREAD_ROUTINE Worker, + IN gctPOINTER Argument, + OUT gctPOINTER * Thread + ); + +/* Close a thread. */ +gceSTATUS +gcoOS_CloseThread( + IN gcoOS Os, + IN gctPOINTER Thread + ); + +/*----------------------------------------------------------------------------*/ +/*----- Mutexes --------------------------------------------------------------*/ + +/* Create a new mutex. */ +gceSTATUS +gcoOS_CreateMutex( + IN gcoOS Os, + OUT gctPOINTER * Mutex + ); + +/* Delete a mutex. */ +gceSTATUS +gcoOS_DeleteMutex( + IN gcoOS Os, + IN gctPOINTER Mutex + ); + +/* Acquire a mutex. */ +gceSTATUS +gcoOS_AcquireMutex( + IN gcoOS Os, + IN gctPOINTER Mutex, + IN gctUINT32 Timeout + ); + +/* Release a mutex. */ +gceSTATUS +gcoOS_ReleaseMutex( + IN gcoOS Os, + IN gctPOINTER Mutex + ); + +/*----------------------------------------------------------------------------*/ +/*----- Signals --------------------------------------------------------------*/ + +/* Create a signal. */ +gceSTATUS +gcoOS_CreateSignal( + IN gcoOS Os, + IN gctBOOL ManualReset, + OUT gctSIGNAL * Signal + ); + +/* Destroy a signal. */ +gceSTATUS +gcoOS_DestroySignal( + IN gcoOS Os, + IN gctSIGNAL Signal + ); + +/* Signal a signal. */ +gceSTATUS +gcoOS_Signal( + IN gcoOS Os, + IN gctSIGNAL Signal, + IN gctBOOL State + ); + +/* Wait for a signal. */ +gceSTATUS +gcoOS_WaitSignal( + IN gcoOS Os, + IN gctSIGNAL Signal, + IN gctUINT32 Wait + ); + +/* Map a signal from another process */ +gceSTATUS +gcoOS_MapSignal( + IN gctSIGNAL RemoteSignal, + OUT gctSIGNAL * LocalSignal + ); + +/* Unmap a signal mapped from another process */ +gceSTATUS +gcoOS_UnmapSignal( + IN gctSIGNAL Signal + ); + +/*----------------------------------------------------------------------------*/ +/*----- Android Native Fence -------------------------------------------------*/ + +/* Create native fence. */ +gceSTATUS +gcoOS_CreateNativeFence( + IN gcoOS Os, + IN gctSIGNAL Signal, + OUT gctINT * FenceFD + ); + +/* (CPU) Wait on native fence. */ +gceSTATUS +gcoOS_ClientWaitNativeFence( + IN gcoOS Os, + IN gctINT FenceFD, + IN gctUINT32 Timeout + ); + +/* (GPU) Wait on native fence. */ +gceSTATUS +gcoOS_WaitNativeFence( + IN gcoOS Os, + IN gctINT FenceFD, + IN gctUINT32 Timeout + ); + +/*----------------------------------------------------------------------------*/ +/*----- Memory Access and Cache ----------------------------------------------*/ + +/* Write a register. */ +gceSTATUS +gcoOS_WriteRegister( + IN gcoOS Os, + IN gctUINT32 Address, + IN gctUINT32 Data + ); + +/* Read a register. */ +gceSTATUS +gcoOS_ReadRegister( + IN gcoOS Os, + IN gctUINT32 Address, + OUT gctUINT32 * Data + ); + +gceSTATUS +gcoOS_CacheClean( + IN gcoOS Os, + IN gctUINT32 Node, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ); + +gceSTATUS +gcoOS_CacheFlush( + IN gcoOS Os, + IN gctUINT32 Node, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ); + +gceSTATUS +gcoOS_CacheInvalidate( + IN gcoOS Os, + IN gctUINT32 Node, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes + ); + +gceSTATUS +gcoOS_MemoryBarrier( + IN gcoOS Os, + IN gctPOINTER Logical + ); + +gceSTATUS +gcoOS_CPUPhysicalToGPUPhysical( + IN gctPHYS_ADDR_T CPUPhysical, + OUT gctPHYS_ADDR_T * GPUPhysical + ); + +gceSTATUS +gcoOS_QuerySystemInfo( + IN gcoOS Os, + OUT gcsSystemInfo *Info + ); + + +/*----------------------------------------------------------------------------*/ +/*----- Profile --------------------------------------------------------------*/ + +gceSTATUS +gckOS_GetProfileTick( + OUT gctUINT64_PTR Tick + ); + +gceSTATUS +gckOS_QueryProfileTickRate( + OUT gctUINT64_PTR TickRate + ); + +gctUINT32 +gckOS_ProfileToMS( + IN gctUINT64 Ticks + ); + +gceSTATUS +gcoOS_GetProfileTick( + OUT gctUINT64_PTR Tick + ); + +gceSTATUS +gcoOS_QueryProfileTickRate( + OUT gctUINT64_PTR TickRate + ); + +#if gcdSTATIC_LINK && defined(WIN32) +void gcoOS_ModuleConstructor( + void + ); + +void gcoOS_ModuleDestructor( + void + ); +#endif + +#define _gcmPROFILE_INIT(prefix, freq, start) \ + do { \ + prefix ## OS_QueryProfileTickRate(&(freq)); \ + prefix ## OS_GetProfileTick(&(start)); \ + } while (gcvFALSE) + +#define _gcmPROFILE_QUERY(prefix, start, ticks) \ + do { \ + prefix ## OS_GetProfileTick(&(ticks)); \ + (ticks) = ((ticks) > (start)) ? ((ticks) - (start)) \ + : (~0ull - (start) + (ticks) + 1); \ + } while (gcvFALSE) + +#if gcdENABLE_PROFILING +# define gcmkPROFILE_INIT(freq, start) _gcmPROFILE_INIT(gck, freq, start) +# define gcmkPROFILE_QUERY(start, ticks) _gcmPROFILE_QUERY(gck, start, ticks) +# define gcmPROFILE_INIT(freq, start) _gcmPROFILE_INIT(gco, freq, start) +# define gcmPROFILE_QUERY(start, ticks) _gcmPROFILE_QUERY(gco, start, ticks) +# define gcmPROFILE_ONLY(x) x +# define gcmPROFILE_ELSE(x) do { } while (gcvFALSE) +# define gcmPROFILE_DECLARE_ONLY(x) x +# define gcmPROFILE_DECLARE_ELSE(x) typedef x +#else +# define gcmkPROFILE_INIT(start, freq) do { } while (gcvFALSE) +# define gcmkPROFILE_QUERY(start, ticks) do { } while (gcvFALSE) +# define gcmPROFILE_INIT(start, freq) do { } while (gcvFALSE) +# define gcmPROFILE_QUERY(start, ticks) do { } while (gcvFALSE) +# define gcmPROFILE_ONLY(x) do { } while (gcvFALSE) +# define gcmPROFILE_ELSE(x) x +# define gcmPROFILE_DECLARE_ONLY(x) do { } while (gcvFALSE) +# define gcmPROFILE_DECLARE_ELSE(x) x +#endif + +/******************************************************************************* +** gcoMATH object +*/ + +#define gcdPI 3.14159265358979323846f + +/* Kernel. */ +gctINT +gckMATH_ModuloInt( + IN gctINT X, + IN gctINT Y + ); + +/* User. */ +gctUINT32 +gcoMATH_Log2in5dot5( + IN gctINT X + ); + + +gctFLOAT +gcoMATH_UIntAsFloat( + IN gctUINT32 X + ); + +gctUINT32 +gcoMATH_FloatAsUInt( + IN gctFLOAT X + ); + +gctBOOL +gcoMATH_CompareEqualF( + IN gctFLOAT X, + IN gctFLOAT Y + ); + +gctUINT16 +gcoMATH_UInt8AsFloat16( + IN gctUINT8 X + ); + +gctUINT32 +gcoMATH_Float16ToFloat( + IN gctUINT16 In + ); + +gctUINT16 +gcoMATH_FloatToFloat16( + IN gctUINT32 In + ); + +gctUINT32 +gcoMATH_Float11ToFloat( + IN gctUINT32 In + ); + +gctUINT16 +gcoMATH_FloatToFloat11( + IN gctUINT32 In + ); + +gctUINT32 +gcoMATH_Float10ToFloat( + IN gctUINT32 In + ); + +gctUINT16 +gcoMATH_FloatToFloat10( + IN gctUINT32 In + ); + +gctUINT32 +gcoMATH_Float14ToFloat( + IN gctUINT16 In + ); + +/******************************************************************************\ +**************************** Coordinate Structures ***************************** +\******************************************************************************/ + +typedef struct _gcsPOINT +{ + gctINT32 x; + gctINT32 y; +} +gcsPOINT; + +typedef struct _gcsSIZE +{ + gctINT32 width; + gctINT32 height; +} +gcsSIZE; + +typedef struct _gcsRECT +{ + gctINT32 left; + gctINT32 top; + gctINT32 right; + gctINT32 bottom; +} +gcsRECT; + +typedef struct _gcsPIXEL +{ + union + { + struct + { + gctFLOAT r, g, b, a; + } f; + struct + { + gctINT32 r, g, b, a; + } i; + struct + { + gctUINT32 r, g, b, a; + } ui; + } color; + + gctFLOAT d; + gctUINT32 s; + +} gcsPIXEL; + +/******************************************************************************\ +********************************* gcoSURF Object ******************************** +\******************************************************************************/ + +/*----------------------------------------------------------------------------*/ +/*------------------------------- gcoSURF Common ------------------------------*/ + +/* Color format component parameters. */ +typedef struct _gcsFORMAT_COMPONENT +{ + gctUINT8 start; + gctUINT8 width; +} +gcsFORMAT_COMPONENT; + +/* RGBA color format class. */ +typedef struct _gcsFORMAT_CLASS_TYPE_RGBA +{ + gcsFORMAT_COMPONENT alpha; + gcsFORMAT_COMPONENT red; + gcsFORMAT_COMPONENT green; + gcsFORMAT_COMPONENT blue; +} +gcsFORMAT_CLASS_TYPE_RGBA; + +/* YUV color format class. */ +typedef struct _gcsFORMAT_CLASS_TYPE_YUV +{ + gcsFORMAT_COMPONENT y; + gcsFORMAT_COMPONENT u; + gcsFORMAT_COMPONENT v; +} +gcsFORMAT_CLASS_TYPE_YUV; + +/* Index color format class. */ +typedef struct _gcsFORMAT_CLASS_TYPE_INDEX +{ + gcsFORMAT_COMPONENT value; +} +gcsFORMAT_CLASS_TYPE_INDEX; + +/* Luminance color format class. */ +typedef struct _gcsFORMAT_CLASS_TYPE_LUMINANCE +{ + gcsFORMAT_COMPONENT alpha; + gcsFORMAT_COMPONENT value; +} +gcsFORMAT_CLASS_TYPE_LUMINANCE; + +/* Bump map color format class. */ +typedef struct _gcsFORMAT_CLASS_TYPE_BUMP +{ + gcsFORMAT_COMPONENT alpha; + gcsFORMAT_COMPONENT l; + gcsFORMAT_COMPONENT v; + gcsFORMAT_COMPONENT u; + gcsFORMAT_COMPONENT q; + gcsFORMAT_COMPONENT w; +} +gcsFORMAT_CLASS_TYPE_BUMP; + +/* Depth and stencil format class. */ +typedef struct _gcsFORMAT_CLASS_TYPE_DEPTH +{ + gcsFORMAT_COMPONENT depth; + gcsFORMAT_COMPONENT stencil; +} +gcsFORMAT_CLASS_TYPE_DEPTH; + +typedef union _gcuPIXEL_FORMAT_CLASS +{ + gcsFORMAT_CLASS_TYPE_BUMP bump; + gcsFORMAT_CLASS_TYPE_RGBA rgba; + gcsFORMAT_CLASS_TYPE_YUV yuv; + gcsFORMAT_CLASS_TYPE_LUMINANCE lum; + gcsFORMAT_CLASS_TYPE_INDEX index; + gcsFORMAT_CLASS_TYPE_DEPTH depth; +} +gcuPIXEL_FORMAT_CLASS; + +/* Format parameters. */ +typedef struct _gcsSURF_FORMAT_INFO +{ + /* Name of the format */ + gctCONST_STRING formatName; + + /* Format code and class. */ + gceSURF_FORMAT format; + gceFORMAT_CLASS fmtClass; + + /* Format data type */ + gceFORMAT_DATATYPE fmtDataType; + + /* The size of one pixel in bits. */ + gctUINT8 bitsPerPixel; + + /* Pixel block dimensions. */ + gctUINT blockWidth; + gctUINT blockHeight; + + /* Pixel block size in bits. */ + gctUINT blockSize; + + /* Some formats are larger than what the GPU can support. */ + /* These formats are read in the number of layers specified. */ + gctUINT8 layers; + + /* The format is faked and software will interpret it differently + ** with HW. Most of them can't be blendable(PE) or filterable(TX). + */ + gctBOOL fakedFormat; + + /* Some formats have two neighbour pixels interleaved together. */ + /* To describe such format, set the flag to 1 and add another */ + /* like this one describing the odd pixel format. */ + gctBOOL interleaved; + + /* sRGB format. */ + gctBOOL sRGB; + + /* How GPU read from big-endian host memory */ + gceENDIAN_HINT endian; + + /* Format components. */ + gcuPIXEL_FORMAT_CLASS u; + + /* Format components. */ + gcuPIXEL_FORMAT_CLASS uOdd; + + /* Render format. */ + gceSURF_FORMAT closestRenderFormat; + /*gctCLOSEST_FORMAT dynamicClosestRenderFormat;*/ + gctUINT renderFormat; + const gceTEXTURE_SWIZZLE * pixelSwizzle; + + /* Texture format. */ + gceSURF_FORMAT closestTXFormat; + gctUINT txFormat; + const gceTEXTURE_SWIZZLE * txSwizzle; + gctBOOL txIntFilter; +} +gcsSURF_FORMAT_INFO; + +/* Frame buffer information. */ +typedef struct _gcsSURF_FRAMEBUFFER +{ + gctPOINTER logical; + gctUINT width, height; + gctINT stride; + gceSURF_FORMAT format; +} +gcsSURF_FRAMEBUFFER; + +/* Generic pixel component descriptors. */ +extern gcsFORMAT_COMPONENT gcvPIXEL_COMP_XXX8; +extern gcsFORMAT_COMPONENT gcvPIXEL_COMP_XX8X; +extern gcsFORMAT_COMPONENT gcvPIXEL_COMP_X8XX; +extern gcsFORMAT_COMPONENT gcvPIXEL_COMP_8XXX; + +/* Construct a new gcoSURF object. */ +gceSTATUS +gcoSURF_Construct( + IN gcoHAL Hal, + IN gctUINT Width, + IN gctUINT Height, + IN gctUINT Depth, + IN gceSURF_TYPE Type, + IN gceSURF_FORMAT Format, + IN gcePOOL Pool, + OUT gcoSURF * Surface + ); + +/* Destroy an gcoSURF object. */ +gceSTATUS +gcoSURF_Destroy( + IN gcoSURF Surface + ); + +/* Map user-allocated surface. */ +gceSTATUS +gcoSURF_MapUserSurface( + IN gcoSURF Surface, + IN gctUINT Alignment, + IN gctPOINTER Logical, + IN gctPHYS_ADDR_T Physical + ); + +/* Wrapp surface with known logical/GPU address */ +gceSTATUS +gcoSURF_WrapSurface( + IN gcoSURF Surface, + IN gctUINT Alignment, + IN gctPOINTER Logical, + IN gctUINT32 Address + ); + + +/* Query vid mem node info. */ +gceSTATUS +gcoSURF_QueryVidMemNode( + IN gcoSURF Surface, + OUT gctUINT32 * Node, + OUT gcePOOL * Pool, + OUT gctSIZE_T_PTR Bytes, + OUT gctUINT32 * TsNode, + OUT gcePOOL * TsPool, + OUT gctSIZE_T_PTR TsBytes + ); + +/* Set the color type of the surface. */ +gceSTATUS +gcoSURF_SetColorType( + IN gcoSURF Surface, + IN gceSURF_COLOR_TYPE ColorType + ); + +/* Get the color type of the surface. */ +gceSTATUS +gcoSURF_GetColorType( + IN gcoSURF Surface, + OUT gceSURF_COLOR_TYPE *ColorType + ); + +/* Set the color space of the surface. */ +gceSTATUS +gcoSURF_SetColorSpace( + IN gcoSURF Surface, + IN gceSURF_COLOR_SPACE ColorSpace + ); + +/* Get the color space of the surface. */ +gceSTATUS +gcoSURF_GetColorSpace( + IN gcoSURF Surface, + OUT gceSURF_COLOR_SPACE *ColorSpace + ); + + +/* Set the surface ration angle. */ +gceSTATUS +gcoSURF_SetRotation( + IN gcoSURF Surface, + IN gceSURF_ROTATION Rotation + ); + +gceSTATUS +gcoSURF_IsValid( + IN gcoSURF Surface + ); + +#if gcdENABLE_3D +/* Verify and return the state of the tile status mechanism. */ +gceSTATUS +gcoSURF_IsTileStatusSupported( + IN gcoSURF Surface + ); + +/* Verify if surface has tile status enabled. */ +gceSTATUS +gcoSURF_IsTileStatusEnabled( + IN gcsSURF_VIEW *SurfView + ); + +/* Verify if surface is compressed. */ +gceSTATUS +gcoSURF_IsCompressed( + IN gcsSURF_VIEW *SurfView + ); + +/* Enable tile status for the specified surface on zero slot. */ +gceSTATUS +gcoSURF_EnableTileStatus( + IN gcsSURF_VIEW *Surface + ); + +/* Enable tile status for the specified surface on specified slot. */ +gceSTATUS +gcoSURF_EnableTileStatusEx( + IN gcsSURF_VIEW *surfView, + IN gctUINT RtIndex + ); + +/* Disable tile status for the specified surface. */ +gceSTATUS +gcoSURF_DisableTileStatus( + IN gcsSURF_VIEW *SurfView, + IN gctBOOL Decompress + ); + +/* Flush tile status cache for the specified surface. */ +gceSTATUS +gcoSURF_FlushTileStatus( + IN gcsSURF_VIEW *SurfView, + IN gctBOOL Decompress + ); +#endif /* gcdENABLE_3D */ + +/* Get surface size. */ +gceSTATUS +gcoSURF_GetSize( + IN gcoSURF Surface, + OUT gctUINT * Width, + OUT gctUINT * Height, + OUT gctUINT * Depth + ); + +/* Get surface information */ +gceSTATUS +gcoSURF_GetInfo( + IN gcoSURF Surface, + IN gceSURF_INFO_TYPE InfoType, + IN OUT gctINT32 *Value + ); + +/* Get surface aligned sizes. */ +gceSTATUS +gcoSURF_GetAlignedSize( + IN gcoSURF Surface, + OUT gctUINT * Width, + OUT gctUINT * Height, + OUT gctINT * Stride + ); + +/* Get alignments. */ +gceSTATUS +gcoSURF_GetAlignment( + IN gceSURF_TYPE Type, + IN gceSURF_FORMAT Format, + OUT gctUINT * AddressAlignment, + OUT gctUINT * XAlignment, + OUT gctUINT * YAlignment + ); + +gceSTATUS +gcoSURF_AlignResolveRect( + IN gcoSURF Surf, + IN gcsPOINT_PTR RectOrigin, + IN gcsPOINT_PTR RectSize, + OUT gcsPOINT_PTR AlignedOrigin, + OUT gcsPOINT_PTR AlignedSize + ); + +/* Get surface type and format. */ +gceSTATUS +gcoSURF_GetFormat( + IN gcoSURF Surface, + OUT OPTIONAL gceSURF_TYPE * Type, + OUT OPTIONAL gceSURF_FORMAT * Format + ); + +/* Get surface information */ +gceSTATUS +gcoSURF_GetFormatInfo( + IN gcoSURF Surface, + OUT gcsSURF_FORMAT_INFO_PTR * formatInfo + ); + +/* Get Surface pack format */ +gceSTATUS +gcoSURF_GetPackedFormat( + IN gcoSURF Surface, + OUT gceSURF_FORMAT * Format + ); + +/* Get surface tiling. */ +gceSTATUS +gcoSURF_GetTiling( + IN gcoSURF Surface, + OUT gceTILING * Tiling + ); + +/* Get bottom buffer offset bytes. */ +gceSTATUS +gcoSURF_GetBottomBufferOffset( + IN gcoSURF Surface, + OUT gctUINT_PTR BottomBufferOffset + ); + +/* Lock the surface. */ +gceSTATUS +gcoSURF_Lock( + IN gcoSURF Surface, + IN OUT gctUINT32 * Address, + IN OUT gctPOINTER * Memory + ); + +/* Unlock the surface. */ +gceSTATUS +gcoSURF_Unlock( + IN gcoSURF Surface, + IN gctPOINTER Memory + ); + +/*. Query surface flags.*/ +gceSTATUS +gcoSURF_QueryFlags( + IN gcoSURF Surface, + IN gceSURF_FLAG Flag + ); + +gceSTATUS +gcoSURF_QueryHints( + IN gcoSURF Surface, + IN gceSURF_TYPE Hints + ); + +/* Return pixel format parameters; Info is required to be a pointer to an + * array of at least two items because some formats have up to two records + * of description. */ +gceSTATUS +gcoSURF_QueryFormat( + IN gceSURF_FORMAT Format, + OUT gcsSURF_FORMAT_INFO_PTR * Info + ); + +/* Compute the color pixel mask. */ +gceSTATUS +gcoSURF_ComputeColorMask( + IN gcsSURF_FORMAT_INFO_PTR Format, + OUT gctUINT32_PTR ColorMask + ); + +/* Flush the surface. */ +gceSTATUS +gcoSURF_Flush( + IN gcoSURF Surface + ); + +/* Fill surface from it's tile status buffer. */ +gceSTATUS +gcoSURF_FillFromTile( + IN gcsSURF_VIEW *SurView + ); + +/* Fill surface with a value. */ +gceSTATUS +gcoSURF_Fill( + IN gcoSURF Surface, + IN gcsPOINT_PTR Origin, + IN gcsSIZE_PTR Size, + IN gctUINT32 Value, + IN gctUINT32 Mask + ); + +/* Alpha blend two surfaces together. */ +gceSTATUS +gcoSURF_Blend( + IN gcoSURF SrcSurf, + IN gcoSURF DstSurf, + IN gcsPOINT_PTR SrcOrigin, + IN gcsPOINT_PTR DstOrigin, + IN gcsSIZE_PTR Size, + IN gceSURF_BLEND_MODE Mode + ); + +/* Create a new gcoSURF wrapper object. */ +gceSTATUS +gcoSURF_ConstructWrapper( + IN gcoHAL Hal, + OUT gcoSURF * Surface + ); + +/* Set surface flags.*/ +gceSTATUS +gcoSURF_SetFlags( + IN gcoSURF Surface, + IN gceSURF_FLAG Flag, + IN gctBOOL Value + ); + +/* Set the underlying buffer for the surface wrapper. */ +gceSTATUS +gcoSURF_SetBuffer( + IN gcoSURF Surface, + IN gceSURF_TYPE Type, + IN gceSURF_FORMAT Format, + IN gctUINT Stride, + IN gctPOINTER Logical, + IN gctUINT64 Physical + ); + +/* Set the size of the surface in pixels and map the underlying buffer. */ +gceSTATUS +gcoSURF_SetWindow( + IN gcoSURF Surface, + IN gctUINT X, + IN gctUINT Y, + IN gctUINT Width, + IN gctUINT Height + ); + +/* Set the size of the surface in pixels and map the underlying buffer. */ +gceSTATUS +gcoSURF_SetImage( + IN gcoSURF Surface, + IN gctUINT X, + IN gctUINT Y, + IN gctUINT Width, + IN gctUINT Height, + IN gctUINT Depth + ); + +/* Set width/height alignment of the surface directly and calculate stride/size. This is only for dri backend now. Please be careful before use. */ +gceSTATUS +gcoSURF_SetAlignment( + IN gcoSURF Surface, + IN gctUINT Width, + IN gctUINT Height + ); + +/* Increase reference count of the surface. */ +gceSTATUS +gcoSURF_ReferenceSurface( + IN gcoSURF Surface + ); + +/* Get surface reference count. */ +gceSTATUS +gcoSURF_QueryReferenceCount( + IN gcoSURF Surface, + OUT gctINT32 * ReferenceCount + ); + +/* Set surface orientation. */ +gceSTATUS +gcoSURF_SetOrientation( + IN gcoSURF Surface, + IN gceORIENTATION Orientation + ); + +/* Query surface orientation. */ +gceSTATUS +gcoSURF_QueryOrientation( + IN gcoSURF Surface, + OUT gceORIENTATION * Orientation + ); + +gceSTATUS +gcoSURF_NODE_Cache( + IN gcsSURF_NODE_PTR Node, + IN gctPOINTER Logical, + IN gctSIZE_T Bytes, + IN gceCACHEOPERATION Operation + ); + +gceSTATUS +gcsSURF_NODE_SetHardwareAddress( + IN gcsSURF_NODE_PTR Node, + IN gctUINT32 Address + ); + +gceSTATUS +gcsSURF_NODE_GetHardwareAddress( + IN gcsSURF_NODE_PTR Node, + OUT gctUINT32_PTR Physical, + OUT gctUINT32_PTR Physical2, + OUT gctUINT32_PTR Physical3, + OUT gctUINT32_PTR PhysicalBottom + ); + +gctUINT32 +gcsSURF_NODE_GetHWAddress( + IN gcsSURF_NODE_PTR Node + ); + +/* Lock and unlock surface node */ +gceSTATUS +gcoSURF_LockNode( + IN gcsSURF_NODE_PTR Node, + OUT gctUINT32 * Address, + OUT gctPOINTER * Memory + ); + +gceSTATUS +gcoSURF_UnLockNode( + IN gcsSURF_NODE_PTR Node, + IN gceSURF_TYPE Type + ); + +/* Perform CPU cache operation on surface node */ +gceSTATUS +gcoSURF_NODE_CPUCacheOperation( + IN gcsSURF_NODE_PTR Node, + IN gceSURF_TYPE Type, + IN gctSIZE_T Offset, + IN gctSIZE_T Length, + IN gceCACHEOPERATION Operation + ); + +/* Perform CPU cache operation on surface */ +gceSTATUS +gcoSURF_CPUCacheOperation( + IN gcoSURF Surface, + IN gceCACHEOPERATION Operation + ); + + +gceSTATUS +gcoSURF_Swap( + IN gcoSURF Surface1, + IN gcoSURF Surface2 + ); + +gceSTATUS +gcoSURF_ResetSurWH( + IN gcoSURF Surface, + IN gctUINT oriw, + IN gctUINT orih, + IN gctUINT alignw, + IN gctUINT alignh, + IN gceSURF_FORMAT fmt +); + +/* Update surface timestamp. */ +gceSTATUS +gcoSURF_UpdateTimeStamp( + IN gcoSURF Surface + ); + +/* Query surface current timestamp. */ +gceSTATUS +gcoSURF_QueryTimeStamp( + IN gcoSURF Surface, + OUT gctUINT64 * TimeStamp + ); + +/* + * Allocate shared buffer for this surface, so that + * surface states can be shared across processes. + */ +gceSTATUS +gcoSURF_AllocShBuffer( + IN gcoSURF Surface, + OUT gctSHBUF * ShBuf + ); + +/* Bind shared buffer to this surface */ +gceSTATUS +gcoSURF_BindShBuffer( + IN gcoSURF Surface, + IN gctSHBUF ShBuf + ); + +/* Push surface shared states to shared buffer. */ +gceSTATUS +gcoSURF_PushSharedInfo( + IN gcoSURF Surface + ); + +/* Pop shared states from shared buffer. */ +gceSTATUS +gcoSURF_PopSharedInfo( + IN gcoSURF Surface + ); + +#if (gcdENABLE_3D) +/* Copy surface. */ +gceSTATUS +gcoSURF_Copy( + IN gcoSURF Surface, + IN gcoSURF Source + ); + +/* Set number of samples for a gcoSURF object. */ +gceSTATUS +gcoSURF_SetSamples( + IN gcoSURF Surface, + IN gctUINT Samples + ); + +/* Get the number of samples per pixel. */ +gceSTATUS +gcoSURF_GetSamples( + IN gcoSURF Surface, + OUT gctUINT_PTR Samples + ); + +/* Append tile status buffer to user pool surface. */ +gceSTATUS +gcoSURF_AppendTileStatus( + IN gcoSURF Surface + ); +#endif + +gceSTATUS +gcoSURF_WrapUserMemory( + IN gcoHAL Hal, + IN gctUINT Width, + IN gctUINT Height, + IN gctUINT Stride, + IN gctUINT Depth, + IN gceSURF_TYPE Type, + IN gceSURF_FORMAT Format, + IN gctUINT32 Handle, + IN gctUINT32 Flag, + OUT gcoSURF * Surface + ); + +gceSTATUS +gcoSURF_WrapUserMultiBuffer( + IN gcoHAL Hal, + IN gctUINT Width, + IN gctUINT Height, + IN gceSURF_TYPE Type, + IN gceSURF_FORMAT Format, + IN gctUINT Stride[3], + IN gctUINT32 Handle[3], + IN gctUINT BufferOffset[3], + IN gctUINT32 Flag, + OUT gcoSURF * Surface + ); + +gceSTATUS +gcoSURF_UpdateMetadata( + IN gcoSURF Surface, + IN gctINT TsFD + ); + +#define MAX_SURF_MIX_SRC_NUM 64 +gceSTATUS +gcoSURF_MixSurfacesCPU( + IN gcoSURF TargetSurface, + IN gctUINT TargetSliceIndex, + IN gcoSURF *SourceSurface, + IN gctUINT *SourceSliceIndices, + IN gctFLOAT *Weights, + IN gctINT Count + ); + +/******************************************************************************\ +******************************* Hash Structure ****************************** +\******************************************************************************/ + +typedef struct _gcsHASH_MD5CTX +{ + gctBOOL bigEndian; + gctSIZE_T bytes; /* Number of bytes processed */ + gctUINT32 states[4]; + gctUINT8 buffer[64]; +} gcsHASH_MD5CTX; + +void gcsHASH_MD5Init( + gcsHASH_MD5CTX *ctx + ); +void gcsHASH_MD5Update( + gcsHASH_MD5CTX *ctx, + const void *data, + gctSIZE_T bytes + ); +void gcsHASH_MD5Final( + gcsHASH_MD5CTX *ctx, + gctUINT8 digest[16] + ); + + +/******************************************************************************\ +******************************* gcsRECT Structure ****************************** +\******************************************************************************/ + +/* Initialize rectangle structure. */ +gceSTATUS +gcsRECT_Set( + OUT gcsRECT_PTR Rect, + IN gctINT32 Left, + IN gctINT32 Top, + IN gctINT32 Right, + IN gctINT32 Bottom + ); + +/* Return the width of the rectangle. */ +gceSTATUS +gcsRECT_Width( + IN gcsRECT_PTR Rect, + OUT gctINT32 * Width + ); + +/* Return the height of the rectangle. */ +gceSTATUS +gcsRECT_Height( + IN gcsRECT_PTR Rect, + OUT gctINT32 * Height + ); + +/* Ensure that top left corner is to the left and above the right bottom. */ +gceSTATUS +gcsRECT_Normalize( + IN OUT gcsRECT_PTR Rect + ); + +/* Compare two rectangles. */ +gceSTATUS +gcsRECT_IsEqual( + IN gcsRECT_PTR Rect1, + IN gcsRECT_PTR Rect2, + OUT gctBOOL * Equal + ); + +/* Compare the sizes of two rectangles. */ +gceSTATUS +gcsRECT_IsOfEqualSize( + IN gcsRECT_PTR Rect1, + IN gcsRECT_PTR Rect2, + OUT gctBOOL * EqualSize + ); + +gceSTATUS +gcsRECT_RelativeRotation( + IN gceSURF_ROTATION Orientation, + IN OUT gceSURF_ROTATION *Relation); + +gceSTATUS + +gcsRECT_Rotate( + + IN OUT gcsRECT_PTR Rect, + + IN gceSURF_ROTATION Rotation, + + IN gceSURF_ROTATION toRotation, + + IN gctINT32 SurfaceWidth, + + IN gctINT32 SurfaceHeight + + ); + +/******************************************************************************\ +**************************** gcsBOUNDARY Structure ***************************** +\******************************************************************************/ + +typedef struct _gcsBOUNDARY +{ + gctINT x; + gctINT y; + gctINT width; + gctINT height; +} +gcsBOUNDARY; + +/******************************************************************************\ +********************************* gcoHEAP Object ******************************** +\******************************************************************************/ + +typedef struct _gcoHEAP * gcoHEAP; + +/* Construct a new gcoHEAP object. */ +gceSTATUS +gcoHEAP_Construct( + IN gcoOS Os, + IN gctSIZE_T AllocationSize, + OUT gcoHEAP * Heap + ); + +/* Destroy an gcoHEAP object. */ +gceSTATUS +gcoHEAP_Destroy( + IN gcoHEAP Heap + ); + +/* Allocate memory. */ +gceSTATUS +gcoHEAP_Allocate( + IN gcoHEAP Heap, + IN gctSIZE_T Bytes, + OUT gctPOINTER * Node + ); + +gceSTATUS +gcoHEAP_GetMemorySize( + IN gcoHEAP Heap, + IN gctPOINTER Memory, + OUT gctSIZE_T_PTR MemorySize + ); + +/* Free memory. */ +gceSTATUS +gcoHEAP_Free( + IN gcoHEAP Heap, + IN gctPOINTER Node + ); + +#if (VIVANTE_PROFILER_SYSTEM_MEMORY || gcdDEBUG) +/* Profile the heap. */ +gceSTATUS +gcoHEAP_ProfileStart( + IN gcoHEAP Heap + ); + +gceSTATUS +gcoHEAP_ProfileEnd( + IN gcoHEAP Heap, + IN gctCONST_STRING Title + ); +#endif + +/******************************************************************************\ +******************************* Debugging Macros ******************************* +\******************************************************************************/ + +void +gcoOS_SetDebugLevel( + IN gctUINT32 Level + ); + +void +gcoOS_GetDebugLevel( + OUT gctUINT32_PTR DebugLevel + ); + +void +gcoOS_GetDebugZone( + IN gctUINT32 Zone, + OUT gctUINT32_PTR DebugZone + ); + +void +gcoOS_SetDebugZone( + IN gctUINT32 Zone + ); + +void +gcoOS_SetDebugFile( + IN gctCONST_STRING FileName + ); + +void +gcoOS_EnableDebugDump( + IN gctBOOL Enable + ); + +gctFILE +gcoOS_ReplaceDebugFile( + IN gctFILE fp + ); + +/******************************************************************************* +** +** gcmFATAL +** +** Print a message to the debugger and execute a break point. +** +** ARGUMENTS: +** +** message Message. +** ... Optional arguments. +*/ + +void +gckOS_DebugFatal( + IN gctCONST_STRING Message, + ... + ); + +void +gcoOS_DebugFatal( + IN gctCONST_STRING Message, + ... + ); + +#if gcmIS_DEBUG(gcdDEBUG_FATAL) +# define gcmFATAL gcoOS_DebugFatal +# define gcmkFATAL gckOS_DebugFatal +#elif gcdHAS_ELLIPSIS +# define gcmFATAL(...) +# define gcmkFATAL(...) +#else + gcmINLINE static void + __dummy_fatal( + IN gctCONST_STRING Message, + ... + ) + { + } +# define gcmFATAL __dummy_fatal +# define gcmkFATAL __dummy_fatal +#endif + +#define gcmENUM2TEXT(e) case e: return #e + +/******************************************************************************* +** +** gcmTRACE +** +** Print a message to the debugfer if the correct level has been set. In +** retail mode this macro does nothing. +** +** ARGUMENTS: +** +** level Level of message. +** message Message. +** ... Optional arguments. +*/ +#define gcvLEVEL_NONE -1 +#define gcvLEVEL_ERROR 0 +#define gcvLEVEL_WARNING 1 +#define gcvLEVEL_INFO 2 +#define gcvLEVEL_VERBOSE 3 + +void +gckOS_DebugTrace( + IN gctUINT32 Level, + IN gctCONST_STRING Message, + ... + ); + +void +gcoOS_DebugTrace( + IN gctUINT32 Level, + IN gctCONST_STRING Message, + ... + ); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) +# define gcmTRACE gcoOS_DebugTrace +# define gcmkTRACE gckOS_DebugTrace +# define gcmkTRACE_N(Level, ArgumentSize, ...) \ + gckOS_DebugTrace(Level, __VA_ARGS__) +#elif gcdHAS_ELLIPSIS +# define gcmTRACE(...) +# define gcmkTRACE(...) +# define gcmkTRACE_N(...) +#else + gcmINLINE static void + __dummy_trace( + IN gctUINT32 Level, + IN gctCONST_STRING Message, + ... + ) + { + } + + gcmINLINE static void + __dummy_trace_n( + IN gctUINT32 Level, + IN gctUINT ArgumentSize, + IN gctCONST_STRING Message, + ... + ) + { + } + +# define gcmTRACE __dummy_trace +# define gcmkTRACE __dummy_trace +# define gcmkTRACE_N __dummy_trace_n +#endif + +/******************************************************************************* +** +** gcmTRACE_ZONE +** +** Print a message to the debugger if the correct level and zone has been +** set. In retail mode this macro does nothing. +** +** ARGUMENTS: +** +** Level Level of message. +** Zone Zone of message. +** Message Message. +** ... Optional arguments. +*/ + +void +gckOS_DebugTraceZone( + IN gctUINT32 Level, + IN gctUINT32 Zone, + IN gctCONST_STRING Message, + ... + ); + +void +gcoOS_DebugTraceZone( + IN gctUINT32 Level, + IN gctUINT32 Zone, + IN gctCONST_STRING Message, + ... + ); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) +# define gcmTRACE_ZONE gcoOS_DebugTraceZone +# define gcmkTRACE_ZONE gckOS_DebugTraceZone +# define gcmkTRACE_ZONE_N(Level, Zone, ArgumentSize, ...) \ + gckOS_DebugTraceZone(Level, Zone, __VA_ARGS__) +#elif gcdHAS_ELLIPSIS +# define gcmTRACE_ZONE(...) +# define gcmkTRACE_ZONE(...) +# define gcmkTRACE_ZONE_N(...) +#else + gcmINLINE static void + __dummy_trace_zone( + IN gctUINT32 Level, + IN gctUINT32 Zone, + IN gctCONST_STRING Message, + ... + ) + { + } + + gcmINLINE static void + __dummy_trace_zone_n( + IN gctUINT32 Level, + IN gctUINT32 Zone, + IN gctUINT ArgumentSize, + IN gctCONST_STRING Message, + ... + ) + { + } + +# define gcmTRACE_ZONE __dummy_trace_zone +# define gcmkTRACE_ZONE __dummy_trace_zone +# define gcmkTRACE_ZONE_N __dummy_trace_zone_n +#endif + + +/******************************************************************************* +** +** gcmDEBUG_ONLY +** +** Execute a statement or function only in DEBUG mode. +** +** ARGUMENTS: +** +** f Statement or function to execute. +*/ +#if gcmIS_DEBUG(gcdDEBUG_CODE) +# define gcmDEBUG_ONLY(f) f +#else +# define gcmDEBUG_ONLY(f) +#endif + + +/******************************************************************************* +** +** gcmSTACK_PUSH +** gcmSTACK_POP +** gcmSTACK_DUMP +** gcmSTACK_REMOVE +** +** Push or pop a function with entry arguments on the trace stack. +** +** ARGUMENTS: +** +** Function Name of function. +** Line Line number. +** Text Optional text. +** ... Optional arguments for text. +** +** Thread Thread id. +*/ +void +gcoOS_StackPush( + IN gctINT8_PTR Identity, + IN gctCONST_STRING Function, + IN gctINT Line, + IN gctCONST_STRING Text, + ... + ); + +void +gcoOS_StackPop( + IN gctINT8_PTR Identity, + IN gctCONST_STRING Function + ); + +void +gcoOS_StackDump( + void); + +void +gcoOS_StackRemove( + IN gctHANDLE Thread + ); + +#if gcmIS_DEBUG(gcdDEBUG_STACK) +# define gcmSTACK_PUSH gcoOS_StackPush +# define gcmSTACK_POP gcoOS_StackPop +# define gcmSTACK_DUMP gcoOS_StackDump +# define gcmSTACK_REMOVE gcoOS_StackRemove +#elif gcdHAS_ELLIPSIS +# define gcmSTACK_PUSH(...) +# define gcmSTACK_POP(...) +# define gcmSTACK_DUMP() +# define gcmSTACK_REMOVE(...) +#else + gcmINLINE static void + __dummy_stack_push( + IN gctCONST_STRING Function, + IN gctINT Line, + IN gctCONST_STRING Text, + ... + ) + { + } + + gcmINLINE static void + __dummy_stack_pop( + IN gctINT8_PTR Identity, + IN gctCONST_STRING Function + ); + + gcmINLINE static void + __dummy_stack_remove( + IN gctHANDLE Thread + ); + +# define gcmSTACK_PUSH __dummy_stack_push +# define gcmSTACK_POP(a,b) __dummy_stack_pop +# define gcmSTACK_DUMP() +# define gcmSTACK_REMOVE(a) __dummy_stack_remove +#endif + + +/******************************************************************************* +** +** gcmBINARY_TRACE +** +** Push or pop a function with entry arguments on the trace stack. +** +** ARGUMENTS: +** +** Function Name of function +** Line Line number +** Text Optional text +** ... Optional arguments for text. +*/ +void +gcoOS_BinaryTrace( + IN gctCONST_STRING Function, + IN gctINT Line, + IN gctCONST_STRING Text OPTIONAL, + ... + ); + +void +gckOS_BinaryTrace( + IN gctCONST_STRING Function, + IN gctINT Line, + IN gctCONST_STRING Text OPTIONAL, + ... + ); + +#if gcdBINARY_TRACE +# define gcmBINARY_TRACE gcoOS_BinaryTrace +# define gcmkBINARY_TRACE gckOS_BinaryTrace +#elif gcdHAS_ELLIPSIS +# define gcmBINARY_TRACE(Function, Line, Text, ...) +# define gcmkBINARY_TRACE(Function, Line, Text, ...) +#else + gcmINLINE static void + __dummy_binary_trace( + IN gctCONST_STRING Function, + IN gctINT Line, + IN gctCONST_STRING Text, + ) + { + } + +# define gcmBINARY_TRACE __dummy_binary_trace +# define gcmkBINARY_TRACE __dummy_binary_trace +#endif + + +/******************************************************************************* +** +** gcmSYSTRACE_BEGIN +** gcmSYSTRACE_END +** +** Systrace is a performance tunning tool on linux. +** +** ARGUMENTS: +** +** FuncName Function name +** Zone Systrace zone. Only specified zones are traced. +*/ + +void +gcoOS_SysTraceBegin( + IN gctUINT32 Zone, + IN gctCONST_STRING FuncName + ); + +void +gcoOS_SysTraceEnd( + IN gctUINT32 Zone + ); + +#if defined(LINUX) && gcdSYSTRACE +# define gcmSYSTRACE_BEGIN gcoOS_SysTraceBegin +# define gcmSYSTRACE_END gcoOS_SysTraceEnd +#elif gcdHAS_ELLIPSIS +# define gcmSYSTRACE_BEGIN(...) +# define gcmSYSTRACE_END(...) +#else + gcmINLINE static void + __dummy_systrace_begin( + IN gctUINT32 Zone, + IN gctCONST_STRING FuncName + ) + { + } + + gcmINLINE static void + __dummy_systrace_end( + IN gctUINT32 Zone + ) + { + } + +# define gcmSYSTRACE_BEGIN __dummy_systrace_begin +# define gcmSYSTRACE_END __dummy_systrace_end +#endif + + +/******************************************************************************\ +******************************** Logging Macros ******************************** +\******************************************************************************/ + +#define gcdHEADER_LEVEL gcvLEVEL_VERBOSE + +/* Always enable header/footer when systrace build is on */ +#if defined(LINUX) && gcdSYSTRACE +#undef gcdEMPTY_HEADER_FOOTER +#endif + +#ifndef gcdEMPTY_HEADER_FOOTER +#define gcdEMPTY_HEADER_FOOTER 0 +#endif + +#if gcdENABLE_PROFILING +void +gcoOS_ProfileDB( + IN gctCONST_STRING Function, + IN OUT gctBOOL_PTR Initialized + ); + +#define gcmHEADER() \ + gctINT8 __user__ = 1; \ + static gctBOOL __profile__initialized__ = gcvFALSE; \ + gcmSTACK_PUSH(&__user__, __FUNCTION__, __LINE__, gcvNULL, gcvNULL); \ + gcoOS_ProfileDB(__FUNCTION__, &__profile__initialized__) + +#define gcmHEADER_ARG(...) \ + gctINT8 __user__ = 1; \ + static gctBOOL __profile__initialized__ = gcvFALSE; \ + gcmSTACK_PUSH(&__user__, __FUNCTION__, __LINE__, Text, __VA_ARGS__); \ + gcoOS_ProfileDB(__FUNCTION__, &__profile__initialized__) + +#define gcmFOOTER() \ + gcmSTACK_POP(&__user__, __FUNCTION__); \ + gcoOS_ProfileDB(__FUNCTION__, gcvNULL) + +#define gcmFOOTER_NO() \ + gcmSTACK_POP(&__user__, __FUNCTION__); \ + gcoOS_ProfileDB(__FUNCTION__, gcvNULL) + +#define gcmFOOTER_ARG(...) \ + gcmSTACK_POP(&__user__, __FUNCTION__); \ + gcoOS_ProfileDB(__FUNCTION__, gcvNULL) + +#define gcmFOOTER_KILL() \ + gcmSTACK_POP(&__user__, __FUNCTION__); \ + gcoOS_ProfileDB(gcvNULL, gcvNULL) + +#else /* !gcdENABLE_PROFILING */ + +#if gcdEMPTY_HEADER_FOOTER +# define gcmHEADER() +#elif gcdHAS_ELLIPSIS +#define gcmHEADER() \ + gctINT8 __user__ = 1; \ + gctINT8_PTR __user_ptr__ = &__user__; \ + gcmSTACK_PUSH(__user_ptr__, __FUNCTION__, __LINE__, gcvNULL, gcvNULL); \ + gcmSYSTRACE_BEGIN(_GC_OBJ_ZONE, __FUNCTION__); \ + gcmBINARY_TRACE(__FUNCTION__, __LINE__, gcvNULL, gcvNULL); \ + gcmTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "++%s(%d)", __FUNCTION__, __LINE__) +#else + gcmINLINE static void + __dummy_header(void) + { + } +# define gcmHEADER __dummy_header +#endif + +#if gcdHAS_ELLIPSIS +#if gcdEMPTY_HEADER_FOOTER +# define gcmHEADER_ARG(Text, ...) +#else +# define gcmHEADER_ARG(Text, ...) \ + gctINT8 __user__ = 1; \ + gctINT8_PTR __user_ptr__ = &__user__; \ + gcmSTACK_PUSH(__user_ptr__, __FUNCTION__, __LINE__, Text, __VA_ARGS__); \ + gcmSYSTRACE_BEGIN(_GC_OBJ_ZONE, __FUNCTION__); \ + gcmBINARY_TRACE(__FUNCTION__, __LINE__, Text, __VA_ARGS__); \ + gcmTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "++%s(%d): " Text, __FUNCTION__, __LINE__, __VA_ARGS__) +#endif +#else + gcmINLINE static void + __dummy_header_arg( + IN gctCONST_STRING Text, + ... + ) + { + } +# define gcmHEADER_ARG __dummy_header_arg +#endif + +#if gcdEMPTY_HEADER_FOOTER +# define gcmFOOTER() +#elif gcdHAS_ELLIPSIS +# define gcmFOOTER() \ + gcmSTACK_POP(__user_ptr__, __FUNCTION__); \ + gcmSYSTRACE_END(_GC_OBJ_ZONE); \ + gcmBINARY_TRACE(__FUNCTION__, __LINE__, gcvNULL, gcvNULL); \ + gcmTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "--%s(%d): status=%d(%s)", \ + __FUNCTION__, __LINE__, \ + status, gcoOS_DebugStatus2Name(status)); \ + *__user_ptr__ -= 1 +#else + gcmINLINE static void + __dummy_footer(void) + { + } +# define gcmFOOTER __dummy_footer +#endif + +#if gcdEMPTY_HEADER_FOOTER +# define gcmFOOTER_NO() +#elif gcdHAS_ELLIPSIS +#define gcmFOOTER_NO() \ + gcmSTACK_POP(__user_ptr__, __FUNCTION__); \ + gcmSYSTRACE_END(_GC_OBJ_ZONE); \ + gcmBINARY_TRACE(__FUNCTION__, __LINE__, gcvNULL, gcvNULL); \ + gcmTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "--%s(%d)", __FUNCTION__, __LINE__); \ + *__user_ptr__ -= 1 +#else + gcmINLINE static void + __dummy_footer_no(void) + { + } +# define gcmFOOTER_NO __dummy_footer_no +#endif + +#if gcdEMPTY_HEADER_FOOTER +# define gcmFOOTER_KILL() +#elif gcdHAS_ELLIPSIS +#define gcmFOOTER_KILL() \ + gcmSTACK_POP(__user_ptr__, __FUNCTION__); \ + gcmSYSTRACE_END(_GC_OBJ_ZONE); \ + gcmBINARY_TRACE(__FUNCTION__, __LINE__, gcvNULL, gcvNULL); \ + gcmTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "--%s(%d)", __FUNCTION__, __LINE__); \ + *__user_ptr__ -= 1 +#else + gcmINLINE static void + __dummy_footer_kill(void) + { + } +# define gcmFOOTER_KILL __dummy_footer_kill +#endif + +#if gcdHAS_ELLIPSIS +#if gcdEMPTY_HEADER_FOOTER +# define gcmFOOTER_ARG(Text, ...) +#else +# define gcmFOOTER_ARG(Text, ...) \ + gcmSTACK_POP(__user_ptr__, __FUNCTION__); \ + gcmSYSTRACE_END(_GC_OBJ_ZONE); \ + gcmBINARY_TRACE(__FUNCTION__, __LINE__, Text, __VA_ARGS__); \ + gcmTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "--%s(%d): " Text, __FUNCTION__, __LINE__, __VA_ARGS__); \ + *__user_ptr__ -= 1 +#endif +#else + gcmINLINE static void + __dummy_footer_arg( + IN gctCONST_STRING Text, + ... + ) + { + } +# define gcmFOOTER_ARG __dummy_footer_arg +#endif + +#endif /* gcdENABLE_PROFILING */ + +#if gcdHAS_ELLIPSIS +#define gcmkHEADER() \ + gctINT8 __kernel__ = 1; \ + gctINT8_PTR __kernel_ptr__ = &__kernel__; \ + gcmkBINARY_TRACE(__FUNCTION__, __LINE__, gcvNULL, gcvNULL); \ + gcmkTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "++%s(%d)", __FUNCTION__, __LINE__) +#else + gcmINLINE static void + __dummy_kheader(void) + { + } +# define gcmkHEADER __dummy_kheader +#endif + +#if gcdHAS_ELLIPSIS +# define gcmkHEADER_ARG(Text, ...) \ + gctINT8 __kernel__ = 1; \ + gctINT8_PTR __kernel_ptr__ = &__kernel__; \ + gcmkBINARY_TRACE(__FUNCTION__, __LINE__, Text, __VA_ARGS__); \ + gcmkTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "++%s(%d): " Text, __FUNCTION__, __LINE__, __VA_ARGS__) +#else + gcmINLINE static void + __dummy_kheader_arg( + IN gctCONST_STRING Text, + ... + ) + { + } +# define gcmkHEADER_ARG __dummy_kheader_arg +#endif + +#if gcdHAS_ELLIPSIS +#define gcmkFOOTER() \ + gcmkBINARY_TRACE(__FUNCTION__, __LINE__, gcvNULL, status); \ + gcmkTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "--%s(%d): status=%d(%s)", \ + __FUNCTION__, __LINE__, status, gckOS_DebugStatus2Name(status)); \ + *__kernel_ptr__ -= 1 +#else + gcmINLINE static void + __dummy_kfooter(void) + { + } +# define gcmkFOOTER __dummy_kfooter +#endif + +#if gcdHAS_ELLIPSIS +#define gcmkFOOTER_NO() \ + gcmkBINARY_TRACE(__FUNCTION__, __LINE__, gcvNULL, gcvNULL); \ + gcmkTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "--%s(%d)", __FUNCTION__, __LINE__); \ + *__kernel_ptr__ -= 1 +#else + gcmINLINE static void + __dummy_kfooter_no(void) + { + } +# define gcmkFOOTER_NO __dummy_kfooter_no +#endif + +#if gcdHAS_ELLIPSIS +# define gcmkFOOTER_ARG(Text, ...) \ + gcmkBINARY_TRACE(__FUNCTION__, __LINE__, Text, __VA_ARGS__); \ + gcmkTRACE_ZONE(gcdHEADER_LEVEL, _GC_OBJ_ZONE, \ + "--%s(%d): " Text, \ + __FUNCTION__, __LINE__, __VA_ARGS__); \ + *__kernel_ptr__ -= 1 +#else + gcmINLINE static void + __dummy_kfooter_arg( + IN gctCONST_STRING Text, + ... + ) + { + } +# define gcmkFOOTER_ARG __dummy_kfooter_arg +#endif + +#define gcmOPT_VALUE(ptr) (((ptr) == gcvNULL) ? 0 : *(ptr)) +#define gcmOPT_VALUE_INDEX(ptr, index) (((ptr) == gcvNULL) ? 0 : ptr[index]) +#define gcmOPT_POINTER(ptr) (((ptr) == gcvNULL) ? gcvNULL : *(ptr)) +#define gcmOPT_STRING(ptr) (((ptr) == gcvNULL) ? "(nil)" : (ptr)) + +void +gckOS_Print( + IN gctCONST_STRING Message, + ... + ); + +void +gcoOS_Print( + IN gctCONST_STRING Message, + ... + ); + +#define gcmPRINT gcoOS_Print +#define gcmkPRINT gckOS_Print +#define gcmkPRINT_N(ArgumentSize, ...) gckOS_Print(__VA_ARGS__) + +#if gcdHAS_ELLIPSIS +# define gcmPRINT_ONCE(Text, ...) \ + { \ + static gctBOOL _once = gcvFALSE; \ + if (!_once) \ + { \ + gcmPRINT(Text, __VA_ARGS__); \ + _once = gcvTRUE; \ + } \ + } \ + +#else + gcmINLINE static void + __dummy_printonce_arg( + IN gctCONST_STRING Text, + ... + ) + { + } +# define gcmPRINT_ONCE __dummy_printonce_arg +#endif + +#if gcdFEATURE_SANITYCHECK +#define gcmFEATURE_CHECKED gcmPRINT_ONCE +#else +#define gcmFEATURE_CHECKED(Text, ...) +#endif + +#if gcdPRINT_VERSION +# define gcmPRINT_VERSION() do { \ + _gcmPRINT_VERSION(gcm); \ + gcmSTACK_DUMP(); \ + } while (0) +# define gcmkPRINT_VERSION() _gcmPRINT_VERSION(gcmk) +# define _gcmPRINT_VERSION(prefix) \ + prefix##TRACE(gcvLEVEL_ERROR, \ + "Vivante HAL version %s", \ + gcvVERSION_STRING) +#else +# define gcmPRINT_VERSION() do { gcmSTACK_DUMP(); } while (gcvFALSE) +# define gcmkPRINT_VERSION() do { } while (gcvFALSE) +#endif + +void +gckOS_Dump( + IN gckOS Os, + IN gctCONST_STRING Format, + ... + ); + +void +gckOS_DumpBuffer( + IN gckOS Os, + IN gceDUMP_BUFFER_TYPE Type, + IN gctPOINTER Buffer, + IN gctUINT64 Address, + IN gctSIZE_T Size + ); + +#if gcdDUMP_IN_KERNEL +# define gcmkDUMP gckOS_Dump + +# define gcmkDUMP_BUFFER gckOS_DumpBuffer +#else +# define gcmkDUMP(...) do {} while (0) +# define gcmkDUMP_BUFFER(...) do {} while (0) +#endif + +/******************************************************************************* +** +** gcmDUMP_FRAMERATE +** +** Print average frame rate +** +*/ +gceSTATUS +gcoOS_DumpFrameRate( + void + ); +# define gcmDUMP_FRAMERATE gcoOS_DumpFrameRate + +/******************************************************************************* +** +** gcoOS_SetDumpFlag +** +** Dump print switch. +** +** ARGUMENTS: +** +** DumpState +** True to enable dump prints. +*/ + +gceSTATUS +gcoOS_SetDumpFlag( + IN gctBOOL DumpState + ); + +/******************************************************************************* +** +** gcmDUMP +** +** Print a dump message. +** +** ARGUMENTS: +** +** gctSTRING Message. +** +** ... Optional arguments. +*/ + +#if gcdDUMP +gceSTATUS +gcoOS_Dump( + IN gcoOS Os, + IN gctCONST_STRING String, + ... + ); +# define gcmDUMP gcoOS_Dump +#else +# define gcmDUMP(...) do {} while (0) +#endif + +/******************************************************************************* +** +** gcmDUMP_BUFFER +** +** Print a buffer to the dump. +** +** ARGUMENTS: +** +** gctSTRING Tag +** Tag for dump. +** +** gctUINT32 Address +** GPU address of buffer. +** +** gctPOINTER Logical +** Logical address of buffer. +** +** gctUINT32 Offset +** Offset into buffer. +** +** gctSIZE_T Bytes +** Number of bytes. +*/ + +#if gcdDUMP +gceSTATUS +gcoOS_DumpBuffer( + IN gcoOS Os, + IN gceDUMP_BUFFER_TYPE Type, + IN gctUINT32 Address, + IN gctPOINTER Logical, + IN gctSIZE_T Offset, + IN gctSIZE_T Bytes + ); +# define gcmDUMP_BUFFER gcoOS_DumpBuffer +#else +# define gcmDUMP_BUFFER(...) do {} while (0) +#endif + +#if gcdDUMP +void +gcoOS_DumpLock( + void + ); +# define gcmDUMP_LOCK gcoOS_DumpLock +#else +# define gcmDUMP_LOCK(...) do {} while (0) +#endif + +#if gcdDUMP +void +gcoOS_DumpUnlock( + void + ); +# define gcmDUMP_UNLOCK gcoOS_DumpUnlock +#else +# define gcmDUMP_UNLOCK(...) do {} while (0) +#endif + +/******************************************************************************* +** +** gcmDUMP_API +** +** Print a dump message for a high level API prefixed by the function name. +** +** ARGUMENTS: +** +** gctSTRING Message. +** +** ... Optional arguments. +*/ +gceSTATUS gcoOS_DumpApi(IN gctCONST_STRING String, ...); +#if gcdDUMP_API +# define gcmDUMP_API gcoOS_DumpApi +#else +# define gcmDUMP_API(...) do {} while (0) +#endif + +/******************************************************************************* +** +** gcmDUMP_API_ARRAY +** +** Print an array of data. +** +** ARGUMENTS: +** +** gctUINT32_PTR Pointer to array. +** gctUINT32 Size. +*/ +gceSTATUS gcoOS_DumpArray(IN gctCONST_POINTER Data, IN gctUINT32 Size); +#if gcdDUMP_API +# define gcmDUMP_API_ARRAY gcoOS_DumpArray +#else +# define gcmDUMP_API_ARRAY(...) do {} while (0) +#endif + +/******************************************************************************* +** +** gcmDUMP_API_ARRAY_TOKEN +** +** Print an array of data terminated by a token. +** +** ARGUMENTS: +** +** gctUINT32_PTR Pointer to array. +** gctUINT32 Termination. +*/ +gceSTATUS gcoOS_DumpArrayToken(IN gctCONST_POINTER Data, IN gctUINT32 Termination); +#if gcdDUMP_API +# define gcmDUMP_API_ARRAY_TOKEN gcoOS_DumpArrayToken +#else +# define gcmDUMP_API_ARRAY_TOKEN(...) do {} while (0) +#endif + +/******************************************************************************* +** +** gcmDUMP_API_DATA +** +** Print an array of bytes. +** +** ARGUMENTS: +** +** gctCONST_POINTER Pointer to array. +** gctSIZE_T Size. +*/ +gceSTATUS gcoOS_DumpApiData(IN gctCONST_POINTER Data, IN gctSIZE_T Size); +#if gcdDUMP_API +# define gcmDUMP_API_DATA gcoOS_DumpApiData +#else +# define gcmDUMP_API_DATA(...) do {} while (0) +#endif + +/******************************************************************************* +** gcmDUMP_2D_COMMAND +** +** Print the 2D command buffer. +** +** ARGUMENTS: +** +** gctUINT32_PTR Pointer to the command buffer. +** gctUINT32 Command buffer size. +*/ +gceSTATUS gcoOS_Dump2DCommand(IN gctUINT32_PTR Command, IN gctUINT32 Size); +#if gcdDUMP_2D +# define gcmDUMP_2D_COMMAND(cmd, size) \ + if (Hardware->newDump2DLevel > 1) \ + gcoOS_Dump2DCommand(cmd, size) +#else +# define gcmDUMP_2D_COMMAND(...) do {} while (0) +#endif + +/******************************************************************************* +** gcmDUMP_2D_SURFACE +** +** Print the 2D surface memory. +** +** ARGUMENTS: +** +** gctBOOL Src. +** gctUINT32 Address. +*/ +gceSTATUS gcoOS_Dump2DSurface(IN gctBOOL Src, IN gctUINT32 Address); +#if gcdDUMP_2D +# define gcmDUMP_2D_SURFACE(src, addr) \ + if (Hardware->newDump2DLevel > 2) \ + gcoOS_Dump2DSurface(src, addr) +#else +# define gcmDUMP_2D_SURFACE(...) do {} while (0) +#endif + +/******************************************************************************* +** gcmDUMP_ADD_MEMORY_INFO +** +** Record the memory info. +** +** ARGUMENTS: +** +** gctUINT32 Address. +** gctSIZE_T Size. +*/ +gceSTATUS gcfAddMemoryInfo(IN gctUINT32 GPUAddress, IN gctPOINTER Logical, IN gctUINT64 Physical, IN gctUINT32 Size); +#if gcdDUMP_2D +# define gcmDUMP_ADD_MEMORY_INFO gcfAddMemoryInfo +#else +# define gcmDUMP_ADD_MEMORY_INFO(...) do {} while (0) +#endif + +/******************************************************************************* +** gcmDUMP_DEL_MEMORY_INFO +** +** Record the memory info. +** +** ARGUMENTS: +** +** gctUINT32 Address. +*/ +gceSTATUS gcfDelMemoryInfo(IN gctUINT32 Address); +#if gcdDUMP_2D +# define gcmDUMP_DEL_MEMORY_INFO gcfDelMemoryInfo +#else +# define gcmDUMP_DEL_MEMORY_INFO(...) do {} while (0) +#endif + +/******************************************************************************* +** +** gcmTRACE_RELEASE +** +** Print a message to the shader debugger. +** +** ARGUMENTS: +** +** message Message. +** ... Optional arguments. +*/ + +#define gcmTRACE_RELEASE gcoOS_DebugShaderTrace + +void +gcoOS_DebugShaderTrace( + IN gctCONST_STRING Message, + ... + ); + +void +gcoOS_SetDebugShaderFiles( + IN gctCONST_STRING VSFileName, + IN gctCONST_STRING FSFileName + ); + +void +gcoOS_SetDebugShaderFileType( + IN gctUINT32 ShaderType + ); + +void +gcoOS_EnableDebugBuffer( + IN gctBOOL Enable + ); + +/******************************************************************************* +** +** gcmBREAK +** +** Break into the debugger. In retail mode this macro does nothing. +** +** ARGUMENTS: +** +** None. +*/ + +void +gcoOS_DebugBreak( + void + ); + +void +gckOS_DebugBreak( + void + ); + +#if gcmIS_DEBUG(gcdDEBUG_BREAK) +# define gcmBREAK gcoOS_DebugBreak +# define gcmkBREAK gckOS_DebugBreak +#else +# define gcmBREAK() +# define gcmkBREAK() +#endif + +/******************************************************************************* +** +** gcmASSERT +** +** Evaluate an expression and break into the debugger if the expression +** evaluates to false. In retail mode this macro does nothing. +** +** ARGUMENTS: +** +** exp Expression to evaluate. +*/ +#if gcmIS_DEBUG(gcdDEBUG_ASSERT) +# define _gcmASSERT(prefix, exp) \ + do \ + { \ + if (!(exp)) \ + { \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "ASSERT at %s(%d)", \ + __FUNCTION__, __LINE__); \ + prefix##TRACE(gcvLEVEL_ERROR, \ + "(%s)", #exp); \ + prefix##BREAK(); \ + } \ + } \ + while (gcvFALSE) +# define gcmASSERT(exp) _gcmASSERT(gcm, exp) +# define gcmkASSERT(exp) _gcmASSERT(gcmk, exp) +#else +# define gcmASSERT(exp) +# define gcmkASSERT(exp) +#endif + +/******************************************************************************* +** +** gcmSTATIC_ASSERT +** +** Tests a software assertion at compile time. If the specific constant +** expression is false, the compiler displays the specified message and +** the compilation fails with an error, otherwise the declaration has +** no effect. +** Static assert is suitable for both user and kernel space. +** +** ARGUMENTS: +** +** exp Constant expression. +** message Error message displayed on assertion failure. +*/ +#if defined(__GNUC__) && ((__GNUC__ == 4 && __GNUC_MINOR__ >= 6) || (__GNUC__ > 4)) +# define gcmSTATIC_ASSERT(constExp, message) \ + do \ + { \ + _Static_assert((constExp), message); \ + } \ + while (0) + +#elif defined(_MSC_VER) && (_MSC_VER >= 1600) +# define gcmSTATIC_ASSERT(constExp, message) \ + static_assert((constExp), message) + +#else +# define gcmSTATIC_ASSERT(constExp, message) \ + do {} while (0) +#endif + +/******************************************************************************* +** +** gcmVERIFY +** +** Verify if an expression returns true. If the expression does not +** evaluates to true, an assertion will happen in debug mode. +** +** ARGUMENTS: +** +** exp Expression to evaluate. +*/ +#if gcmIS_DEBUG(gcdDEBUG_ASSERT) +# define gcmVERIFY(exp) gcmASSERT(exp) +# define gcmkVERIFY(exp) gcmkASSERT(exp) +#else +# define gcmVERIFY(exp) (void)exp +# define gcmkVERIFY(exp) (void)exp +#endif + +/******************************************************************************* +** +** gcmVERIFY_OK +** +** Verify a fucntion returns gcvSTATUS_OK. If the function does not return +** gcvSTATUS_OK, an assertion will happen in debug mode. +** +** ARGUMENTS: +** +** func Function to evaluate. +*/ + +void +gcoOS_Verify( + IN gceSTATUS status + ); + +void +gckOS_Verify( + IN gceSTATUS status + ); + +#if gcmIS_DEBUG(gcdDEBUG_ASSERT) +# define gcmVERIFY_OK(func) \ + do \ + { \ + gceSTATUS verifyStatus = func; \ + gcoOS_Verify(verifyStatus); \ + if (verifyStatus != gcvSTATUS_OK) \ + { \ + gcmTRACE(\ + gcvLEVEL_ERROR, \ + "gcmVERIFY_OK(%d): function returned %d", \ + __LINE__, verifyStatus \ + ); \ + } \ + gcmASSERT(verifyStatus == gcvSTATUS_OK); \ + } \ + while (gcvFALSE) +# define gcmkVERIFY_OK(func) \ + do \ + { \ + gceSTATUS verifyStatus = func; \ + if (verifyStatus != gcvSTATUS_OK) \ + { \ + gcmkTRACE(\ + gcvLEVEL_ERROR, \ + "gcmkVERIFY_OK(%d): function returned %d", \ + __LINE__, verifyStatus \ + ); \ + } \ + gckOS_Verify(verifyStatus); \ + gcmkASSERT(verifyStatus == gcvSTATUS_OK); \ + } \ + while (gcvFALSE) +#else +# define gcmVERIFY_OK(func) func +# define gcmkVERIFY_OK(func) func +#endif + +gctCONST_STRING +gcoOS_DebugStatus2Name( + gceSTATUS status + ); + +gctCONST_STRING +gckOS_DebugStatus2Name( + gceSTATUS status + ); + +/******************************************************************************* +** +** gcmERR_BREAK +** +** Executes a break statement on error. +** +** ASSUMPTIONS: +** +** 'status' variable of gceSTATUS type must be defined. +** +** ARGUMENTS: +** +** func Function to evaluate. +*/ +#define _gcmERR_BREAK(prefix, func){ \ + status = func; \ + if (gcmIS_ERROR(status)) \ + { \ + prefix##PRINT_VERSION(); \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "ERR_BREAK: status=%d(%s) @ %s(%d)", \ + status, gcoOS_DebugStatus2Name(status), __FUNCTION__, __LINE__); \ + break; \ + } \ + do { } while (gcvFALSE); \ + } + +#define _gcmkERR_BREAK(prefix, func){ \ + status = func; \ + if (gcmIS_ERROR(status)) \ + { \ + prefix##PRINT_VERSION(); \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "ERR_BREAK: status=%d(%s) @ %s(%d)", \ + status, gckOS_DebugStatus2Name(status), __FUNCTION__, __LINE__); \ + break; \ + } \ + do { } while (gcvFALSE); \ + } + +#define gcmERR_BREAK(func) _gcmERR_BREAK(gcm, func) +#define gcmkERR_BREAK(func) _gcmkERR_BREAK(gcmk, func) + +/******************************************************************************* +** +** gcmERR_RETURN +** +** Executes a return on error. +** +** ASSUMPTIONS: +** +** 'status' variable of gceSTATUS type must be defined. +** +** ARGUMENTS: +** +** func Function to evaluate. +*/ +#define _gcmERR_RETURN(prefix, func) \ + status = func; \ + if (gcmIS_ERROR(status)) \ + { \ + prefix##PRINT_VERSION(); \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "ERR_RETURN: status=%d(%s) @ %s(%d)", \ + status, gcoOS_DebugStatus2Name(status), __FUNCTION__, __LINE__); \ + prefix##FOOTER(); \ + return status; \ + } \ + do { } while (gcvFALSE) +#define _gcmkERR_RETURN(prefix, func) \ + status = func; \ + if (gcmIS_ERROR(status)) \ + { \ + prefix##PRINT_VERSION(); \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "ERR_RETURN: status=%d(%s) @ %s(%d)", \ + status, gckOS_DebugStatus2Name(status), __FUNCTION__, __LINE__); \ + prefix##FOOTER(); \ + return status; \ + } \ + do { } while (gcvFALSE) +#define gcmERR_RETURN(func) _gcmERR_RETURN(gcm, func) +#define gcmkERR_RETURN(func) _gcmkERR_RETURN(gcmk, func) + + +/******************************************************************************* +** +** gcmONERROR +** +** Jump to the error handler in case there is an error. +** +** ASSUMPTIONS: +** +** 'status' variable of gceSTATUS type must be defined. +** +** ARGUMENTS: +** +** func Function to evaluate. +*/ +#define _gcmONERROR(prefix, func) \ + do \ + { \ + status = func; \ + if (gcmIS_ERROR(status)) \ + { \ + prefix##PRINT_VERSION(); \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "ONERROR: status=%d(%s) @ %s(%d)", \ + status, gcoOS_DebugStatus2Name(status), __FUNCTION__, __LINE__); \ + goto OnError; \ + } \ + } \ + while (gcvFALSE) +#define _gcmkONERROR(prefix, func) \ + do \ + { \ + status = func; \ + if (gcmIS_ERROR(status)) \ + { \ + prefix##PRINT_VERSION(); \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "ONERROR: status=%d(%s) @ %s(%d)", \ + status, gckOS_DebugStatus2Name(status), __FUNCTION__, __LINE__); \ + goto OnError; \ + } \ + } \ + while (gcvFALSE) +#define gcmONERROR(func) _gcmONERROR(gcm, func) +#define gcmkONERROR(func) _gcmkONERROR(gcmk, func) + +#define gcmGET_INDEX_SIZE(type, size) \ + switch (type) \ + { \ + case gcvINDEX_8: \ + size = 1; \ + break; \ + case gcvINDEX_16: \ + size = 2; \ + break; \ + case gcvINDEX_32: \ + size = 4; \ + break; \ + default: \ + gcmONERROR(gcvSTATUS_INVALID_ARGUMENT); \ + } \ + +/******************************************************************************* +** +** gcmkSAFECASTSIZET +** +** Check wether value of a gctSIZE_T varible beyond the capability +** of 32bits GPU hardware. +** +** ASSUMPTIONS: +** +** +** +** ARGUMENTS: +** +** x A gctUINT32 variable +** y A gctSIZE_T variable +*/ +#define gcmkSAFECASTSIZET(x, y) \ + do \ + { \ + gctUINT32 tmp = (gctUINT32)(y); \ + if (gcmSIZEOF(gctSIZE_T) > gcmSIZEOF(gctUINT32)) \ + { \ + gcmkASSERT(tmp <= gcvMAXUINT32); \ + } \ + (x) = tmp; \ + } \ + while (gcvFALSE) + +#define gcmSAFECASTSIZET(x, y) \ + do \ + { \ + gctUINT32 tmp = (gctUINT32)(y); \ + if (gcmSIZEOF(gctSIZE_T) > gcmSIZEOF(gctUINT32)) \ + { \ + gcmASSERT(tmp <= gcvMAXUINT32); \ + } \ + (x) = tmp; \ + } \ + while (gcvFALSE) + +/******************************************************************************* +** +** gcmkSAFECASTPHYSADDRT +** +** Check whether value of a gctPHYS_ADDR_T variable beyond the capability +** of 32bits GPU hardware. +** +** ASSUMPTIONS: +** +** +** +** ARGUMENTS: +** +** x A gctUINT32 variable +** y A gctPHYS_ADDR_T variable +*/ +#define gcmkSAFECASTPHYSADDRT(x, y) \ + do \ + { \ + gctUINT32 tmp = (gctUINT32)(y); \ + if (gcmSIZEOF(gctPHYS_ADDR_T) > gcmSIZEOF(gctUINT32)) \ + { \ + gcmkASSERT(tmp <= gcvMAXUINT32); \ + } \ + (x) = tmp; \ + } \ + while (gcvFALSE) + +/******************************************************************************* +** +** gcmSAFECASTPHYSADDRT +** +** Check whether value of a gctPHYS_ADDR_T variable beyond the capability +** of 32bits GPU hardware. +** +** ASSUMPTIONS: +** +** +** +** ARGUMENTS: +** +** x A gctUINT32 variable +** y A gctPHYS_ADDR_T variable +*/ +#define gcmSAFECASTPHYSADDRT(x, y) \ + do \ + { \ + gctUINT32 tmp = (gctUINT32)(y); \ + if (gcmSIZEOF(gctPHYS_ADDR_T) > gcmSIZEOF(gctUINT32)) \ + { \ + gcmASSERT(tmp <= gcvMAXUINT32); \ + } \ + (x) = tmp; \ + } \ + while (gcvFALSE) + +/******************************************************************************* +** +** gcmVERIFY_LOCK +** +** Verifies whether the surface is locked. +** +** ARGUMENTS: +** +** surfaceInfo Pointer to the surface iniformational structure. +*/ +#define gcmVERIFY_LOCK(surfaceInfo) \ + if (!surfaceInfo->node.valid) \ + { \ + gcmONERROR(gcvSTATUS_MEMORY_UNLOCKED); \ + } \ + +/******************************************************************************* +** +** gcmVERIFY_NODE_LOCK +** +** Verifies whether the surface node is locked. +** +** ARGUMENTS: +** +** surfaceInfo Pointer to the surface iniformational structure. +*/ +#define gcmVERIFY_NODE_LOCK(surfaceNode) \ + if (!(surfaceNode)->valid) \ + { \ + status = gcvSTATUS_MEMORY_UNLOCKED; \ + break; \ + } \ + do { } while (gcvFALSE) + +/******************************************************************************* +** +** gcmBADOBJECT_BREAK +** +** Executes a break statement on bad object. +** +** ARGUMENTS: +** +** obj Object to test. +** t Expected type of the object. +*/ +#define gcmBADOBJECT_BREAK(obj, t) \ + if ((obj == gcvNULL) \ + || (((gcsOBJECT *)(obj))->type != t) \ + ) \ + { \ + status = gcvSTATUS_INVALID_OBJECT; \ + break; \ + } \ + do { } while (gcvFALSE) + +/******************************************************************************* +** +** gcmCHECK_STATUS +** +** Executes a break statement on error. +** +** ASSUMPTIONS: +** +** 'status' variable of gceSTATUS type must be defined. +** +** ARGUMENTS: +** +** func Function to evaluate. +*/ +#define _gcmCHECK_STATUS(prefix, func) \ + do \ + { \ + last = func; \ + if (gcmIS_ERROR(last)) \ + { \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "CHECK_STATUS: status=%d(%s) @ %s(%d)", \ + last, gcoOS_DebugStatus2Name(last), __FUNCTION__, __LINE__); \ + status = last; \ + } \ + } \ + while (gcvFALSE) +#define _gcmkCHECK_STATUS(prefix, func) \ + do \ + { \ + last = func; \ + if (gcmIS_ERROR(last)) \ + { \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "CHECK_STATUS: status=%d(%s) @ %s(%d)", \ + last, gckOS_DebugStatus2Name(last), __FUNCTION__, __LINE__); \ + status = last; \ + } \ + } \ + while (gcvFALSE) +#define gcmCHECK_STATUS(func) _gcmCHECK_STATUS(gcm, func) +#define gcmkCHECK_STATUS(func) _gcmkCHECK_STATUS(gcmk, func) + +/******************************************************************************* +** +** gcmVERIFY_ARGUMENT +** +** Assert if an argument does not apply to the specified expression. If +** the argument evaluates to false, gcvSTATUS_INVALID_ARGUMENT will be +** returned from the current function. In retail mode this macro does +** nothing. +** +** ARGUMENTS: +** +** arg Argument to evaluate. +*/ +# define _gcmVERIFY_ARGUMENT(prefix, arg) \ + do \ + { \ + if (!(arg)) \ + { \ + prefix##TRACE(gcvLEVEL_ERROR, #prefix "VERIFY_ARGUMENT failed:"); \ + prefix##ASSERT(arg); \ + prefix##FOOTER_ARG("status=%d", gcvSTATUS_INVALID_ARGUMENT); \ + return gcvSTATUS_INVALID_ARGUMENT; \ + } \ + } \ + while (gcvFALSE) +# define gcmVERIFY_ARGUMENT(arg) _gcmVERIFY_ARGUMENT(gcm, arg) +# define gcmkVERIFY_ARGUMENT(arg) _gcmVERIFY_ARGUMENT(gcmk, arg) + +/******************************************************************************* +** +** gcmDEBUG_VERIFY_ARGUMENT +** +** Works just like gcmVERIFY_ARGUMENT, but is only valid in debug mode. +** Use this to verify arguments inside non-public API functions. +*/ +#if gcdDEBUG +# define gcmDEBUG_VERIFY_ARGUMENT(arg) _gcmVERIFY_ARGUMENT(gcm, arg) +# define gcmkDEBUG_VERIFY_ARGUMENT(arg) _gcmkVERIFY_ARGUMENT(gcm, arg) +#else +# define gcmDEBUG_VERIFY_ARGUMENT(arg) +# define gcmkDEBUG_VERIFY_ARGUMENT(arg) +#endif + +/******************************************************************************* +** +** gcmVERIFY_ARGUMENT_RETURN +** +** Assert if an argument does not apply to the specified expression. If +** the argument evaluates to false, gcvSTATUS_INVALID_ARGUMENT will be +** returned from the current function. In retail mode this macro does +** nothing. +** +** ARGUMENTS: +** +** arg Argument to evaluate. +*/ +# define _gcmVERIFY_ARGUMENT_RETURN(prefix, arg, value) \ + do \ + { \ + if (!(arg)) \ + { \ + prefix##TRACE(gcvLEVEL_ERROR, \ + #prefix "gcmVERIFY_ARGUMENT_RETURN failed:"); \ + prefix##ASSERT(arg); \ + prefix##FOOTER_ARG("value=%d", value); \ + return value; \ + } \ + } \ + while (gcvFALSE) +# define gcmVERIFY_ARGUMENT_RETURN(arg, value) \ + _gcmVERIFY_ARGUMENT_RETURN(gcm, arg, value) +# define gcmkVERIFY_ARGUMENT_RETURN(arg, value) \ + _gcmVERIFY_ARGUMENT_RETURN(gcmk, arg, value) + +#define MAX_LOOP_COUNT 0x7FFFFFFF + +/******************************************************************************\ +****************************** User Debug Option ****************************** +\******************************************************************************/ + +typedef struct _gcsUSER_DEBUG_OPTION +{ + gceDEBUG_MSG debugMsg; +} +gcsUSER_DEBUG_OPTION; + +gcsUSER_DEBUG_OPTION * +gcoHAL_GetUserDebugOption( + void + ); + +#if gcdHAS_ELLIPSIS +#define gcmUSER_DEBUG_MSG(level, ...) \ + do \ + { \ + if (level <= gcoHAL_GetUserDebugOption()->debugMsg) \ + { \ + gcoOS_Print(__VA_ARGS__); \ + } \ + } while (gcvFALSE) + +#define gcmUSER_DEBUG_ERROR_MSG(...) gcmUSER_DEBUG_MSG(gcvDEBUG_MSG_ERROR, "Error: " __VA_ARGS__) +#define gcmUSER_DEBUG_WARNING_MSG(...) gcmUSER_DEBUG_MSG(gcvDEBUG_MSG_WARNING, "Warring: " __VA_ARGS__) +#else +#define gcmUSER_DEBUG_MSG +#define gcmUSER_DEBUG_ERROR_MSG +#define gcmUSER_DEBUG_WARNING_MSG +#endif + +/******************************************************************************* +** +** A set of macros to aid state loading. +** +** ARGUMENTS: +** +** CommandBuffer Pointer to a gcoCMDBUF object. +** StateDelta Pointer to a gcsSTATE_DELTA state delta structure. +** Memory Destination memory pointer of gctUINT32_PTR type. +** PartOfContext Whether or not the state is a part of the context. +** FixedPoint Whether or not the state is of the fixed point format. +** Count Number of consecutive states to be loaded. +** Address State address. +** Data Data to be set to the state. +*/ + +/*----------------------------------------------------------------------------*/ + +#if gcmIS_DEBUG(gcdDEBUG_CODE) + +# define gcmSTORELOADSTATE(CommandBuffer, Memory, Address, Count) \ + CommandBuffer->lastLoadStatePtr = gcmPTR_TO_UINT64(Memory); \ + CommandBuffer->lastLoadStateAddress = Address; \ + CommandBuffer->lastLoadStateCount = Count + +# define gcmVERIFYLOADSTATE(CommandBuffer, Memory, Address) \ + gcmASSERT(\ + (gctUINT) (Memory - gcmUINT64_TO_TYPE(CommandBuffer->lastLoadStatePtr, gctUINT32_PTR) - 1) \ + == \ + (gctUINT) (Address - CommandBuffer->lastLoadStateAddress) \ + ); \ + \ + gcmASSERT(CommandBuffer->lastLoadStateCount > 0); \ + \ + CommandBuffer->lastLoadStateCount -= 1 + +# define gcmVERIFYLOADSTATEDONE(CommandBuffer) \ + gcmASSERT(CommandBuffer->lastLoadStateCount == 0); + +# define gcmDEFINELOADSTATEBASE() \ + gctUINT32_PTR LoadStateBase; + +# define gcmSETLOADSTATEBASE(CommandBuffer, OutSide) \ + if (OutSide) \ + {\ + LoadStateBase = (gctUINT32_PTR)*OutSide; \ + }\ + else\ + {\ + LoadStateBase = (gctUINT_PTR)CommandBuffer->buffer;\ + } + + +# define gcmVERIFYLOADSTATEALIGNED(CommandBuffer, Memory) \ + gcmASSERT(((Memory - LoadStateBase) & 1) == 0); + +# define gcmUNSETLOADSTATEBASE() \ + LoadStateBase = LoadStateBase; + +#else + +# define gcmSTORELOADSTATE(CommandBuffer, Memory, Address, Count) +# define gcmVERIFYLOADSTATE(CommandBuffer, Memory, Address) +# define gcmVERIFYLOADSTATEDONE(CommandBuffer) + +# define gcmDEFINELOADSTATEBASE() +# define gcmSETLOADSTATEBASE(CommandBuffer, OutSide) +# define gcmVERIFYLOADSTATEALIGNED(CommandBuffer, Memory) +# define gcmUNSETLOADSTATEBASE() + +#endif + +/*----------------------------------------------------------------------------*/ + +/* This style of dump is deprecated. */ +# define gcmDUMPSTATEDATA(StateDelta, FixedPoint, Address, Data) + +#define gcmDEFINESTATEBUFFER(CommandBuffer, StateDelta, Memory, ReserveSize) \ + gctSIZE_T ReserveSize; \ + gcoCMDBUF CommandBuffer; \ + gctUINT32_PTR Memory; \ + gcsSTATE_DELTA_PTR StateDelta; \ + gceENGINE CurrentEngine = gcvENGINE_RENDER + +#define gcmBEGINSTATEBUFFER(Hardware, CommandBuffer, StateDelta, Memory, ReserveSize) \ +{ \ + gcmONERROR(gcoBUFFER_Reserve(\ + Hardware->engine[CurrentEngine].buffer, ReserveSize, gcvTRUE, gcvCOMMAND_3D, &CommandBuffer \ + )); \ + \ + Memory = (gctUINT32_PTR) gcmUINT64_TO_PTR(CommandBuffer->lastReserve); \ + \ + StateDelta = Hardware->delta; \ + \ +} + +#define gcmENDSTATEBUFFER(Hardware, CommandBuffer, Memory, ReserveSize) \ +{ \ + gcmASSERT(\ + gcmUINT64_TO_TYPE(CommandBuffer->lastReserve, gctUINT8_PTR) + ReserveSize \ + == \ + (gctUINT8_PTR) Memory \ + ); \ +} + +/*----------------------------------------------------------------------------*/ + +#define gcmBEGINSTATEBATCH(CommandBuffer, Memory, FixedPoint, Address, Count) \ +{ \ + gcmASSERT(((Memory - gcmUINT64_TO_TYPE(CommandBuffer->lastReserve, gctUINT32_PTR)) & 1) == 0); \ + gcmASSERT((gctUINT32)Count <= 1024); \ + \ + gcmVERIFYLOADSTATEDONE(CommandBuffer); \ + \ + gcmSTORELOADSTATE(CommandBuffer, Memory, Address, Count); \ + \ + *Memory++ \ + = gcmSETFIELDVALUE(0, AQ_COMMAND_LOAD_STATE_COMMAND, OPCODE, LOAD_STATE) \ + | gcmSETFIELD (0, AQ_COMMAND_LOAD_STATE_COMMAND, FLOAT, FixedPoint) \ + | gcmSETFIELD (0, AQ_COMMAND_LOAD_STATE_COMMAND, COUNT, Count) \ + | gcmSETFIELD (0, AQ_COMMAND_LOAD_STATE_COMMAND, ADDRESS, Address); \ +} + +#define gcmENDSTATEBATCH(CommandBuffer, Memory) \ +{ \ + gcmVERIFYLOADSTATEDONE(CommandBuffer); \ + \ + gcmASSERT(((Memory - gcmUINT64_TO_TYPE(CommandBuffer->lastReserve, gctUINT32_PTR)) & 1) == 0); \ +} + +/*----------------------------------------------------------------------------*/ + +#define gcmSETSTATEDATA(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + gcmVERIFYLOADSTATE(CommandBuffer, Memory, Address); \ + \ + gcmSAFECASTSIZET(__temp_data32__, Data); \ + \ + *Memory++ = __temp_data32__; \ + \ + gcoHARDWARE_UpdateDelta(\ + StateDelta, Address, 0, __temp_data32__ \ + ); \ + \ + gcmDUMPSTATEDATA(StateDelta, FixedPoint, Address, __temp_data32__); \ +} + +#define gcmSETSTATEDATAWITHMASK(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + gcmVERIFYLOADSTATE(CommandBuffer, Memory, Address); \ + \ + __temp_data32__ = Data; \ + \ + *Memory++ = __temp_data32__; \ + \ + gcoHARDWARE_UpdateDelta(\ + StateDelta, Address, Mask, __temp_data32__ \ + ); \ + \ + gcmDUMPSTATEDATA(StateDelta, FixedPoint, Address, __temp_data32__); \ +} + + +#define gcmSETCTRLSTATE(StateDelta, CommandBuffer, Memory, Address, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + gcmVERIFYLOADSTATE(CommandBuffer, Memory, Address); \ + \ + __temp_data32__ = Data; \ + \ + *Memory++ = __temp_data32__; \ + \ + gcmDUMPSTATEDATA(StateDelta, gcvFALSE, Address, __temp_data32__); \ +} + +#define gcmSETFILLER(CommandBuffer, Memory) \ +{ \ + gcmVERIFYLOADSTATEDONE(CommandBuffer); \ + \ + *(gctUINT32_PTR)Memory = 0x18000000; \ + Memory += 1; \ +} + +/*----------------------------------------------------------------------------*/ + +#define gcmSETSINGLESTATE(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gcmBEGINSTATEBATCH(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETSTATEDATA(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data); \ + gcmENDSTATEBATCH(CommandBuffer, Memory); \ +} + +#define gcmSETSINGLESTATEWITHMASK(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data) \ +{ \ + gcmBEGINSTATEBATCH(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETSTATEDATAWITHMASK(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data); \ + gcmENDSTATEBATCH(CommandBuffer, Memory); \ +} + + +#define gcmSETSINGLECTRLSTATE(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gcmBEGINSTATEBATCH(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETCTRLSTATE(StateDelta, CommandBuffer, Memory, Address, Data); \ + gcmENDSTATEBATCH(CommandBuffer, Memory); \ +} + + + +#define gcmSETSEMASTALLPIPE(StateDelta, CommandBuffer, Memory, Data) \ +{ \ + gcmSETSINGLECTRLSTATE(StateDelta, CommandBuffer, Memory, gcvFALSE, 0x0E02, Data); \ + \ + *Memory++ = gcmSETFIELDVALUE(0, STALL_COMMAND, OPCODE, STALL); \ + \ + *Memory++ = Data; \ + \ + gcmDUMP(gcvNULL, "#[stall 0x%08X 0x%08X]", \ + gcmSETFIELDVALUE(0, AQ_SEMAPHORE, SOURCE, FRONT_END), \ + gcmSETFIELDVALUE(0, AQ_SEMAPHORE, DESTINATION, PIXEL_ENGINE)); \ +} + +/******************************************************************************* +** +** gcmSETSTARTDECOMMAND +** +** Form a START_DE command. +** +** ARGUMENTS: +** +** Memory Destination memory pointer of gctUINT32_PTR type. +** Count Number of the rectangles. +*/ + +#define gcmSETSTARTDECOMMAND(Memory, Count) \ +{ \ + *Memory++ \ + = gcmSETFIELDVALUE(0, AQ_COMMAND_START_DE_COMMAND, OPCODE, START_DE) \ + | gcmSETFIELD (0, AQ_COMMAND_START_DE_COMMAND, COUNT, Count) \ + | gcmSETFIELD (0, AQ_COMMAND_START_DE_COMMAND, DATA_COUNT, 0); \ + \ + *Memory++ = 0xDEADDEED; \ +} + +/***************************************** +** Temp command buffer macro +*/ +#define gcmDEFINESTATEBUFFER_NEW(CommandBuffer, StateDelta, Memory) \ + gcmDEFINELOADSTATEBASE() \ + gcsTEMPCMDBUF CommandBuffer = gcvNULL; \ + gctUINT32_PTR Memory; \ + gcsSTATE_DELTA_PTR StateDelta; \ + gceENGINE CurrentEngine = gcvENGINE_RENDER + + +#define gcmBEGINSTATEBUFFER_NEW(Hardware, CommandBuffer, StateDelta, Memory, OutSide) \ +{ \ + if (OutSide) \ + {\ + Memory = (gctUINT32_PTR)*OutSide; \ + }\ + else \ + {\ + gcmONERROR(gcoBUFFER_StartTEMPCMDBUF(\ + Hardware->engine[CurrentEngine].buffer, Hardware->engine[CurrentEngine].queue, &CommandBuffer \ + ));\ + \ + Memory = (gctUINT32_PTR)(CommandBuffer->buffer); \ + \ + }\ + StateDelta = Hardware->tempDelta; \ + \ + gcmSETLOADSTATEBASE(CommandBuffer,OutSide);\ +} + +#define gcmENDSTATEBUFFER_NEW(Hardware, CommandBuffer, Memory, OutSide) \ +{ \ + if (OutSide) \ + {\ + *OutSide = Memory; \ + }\ + else \ + {\ + CommandBuffer->currentByteSize = (gctUINT32)((gctUINT8_PTR)Memory - \ + (gctUINT8_PTR)CommandBuffer->buffer); \ + \ + gcmONERROR(gcoBUFFER_EndTEMPCMDBUF(Hardware->engine[CurrentEngine].buffer, gcvFALSE));\ + if (Hardware->constructType != gcvHARDWARE_2D) \ + { \ + gcoHARDWARE_UpdateTempDelta(Hardware);\ + } \ + }\ + gcmUNSETLOADSTATEBASE()\ +} + +#define gcmDEFINECTRLSTATEBUFFER(CommandBuffer, Memory) \ + gcmDEFINELOADSTATEBASE() \ + gcsTEMPCMDBUF CommandBuffer = gcvNULL; \ + gctUINT32_PTR Memory; \ + gceENGINE CurrentEngine = gcvENGINE_RENDER + +#define gcmBEGINCTRLSTATEBUFFER(Hardware, CommandBuffer, Memory, OutSide) \ +{ \ + if (OutSide) \ + { \ + Memory = (gctUINT32_PTR)*OutSide; \ + } \ + else \ + { \ + gcmONERROR(gcoBUFFER_StartTEMPCMDBUF(\ + Hardware->engine[CurrentEngine].buffer, \ + Hardware->engine[CurrentEngine].queue, &CommandBuffer \ + )); \ + \ + Memory = (gctUINT32_PTR)(CommandBuffer->buffer); \ + } \ + gcmSETLOADSTATEBASE(CommandBuffer,OutSide); \ +} + +/*----------------------------------------------------------------------------*/ + +#define gcmBEGINSTATEBATCH_NEW(CommandBuffer, Memory, FixedPoint, Address, Count) \ +{ \ + gcmVERIFYLOADSTATEALIGNED(CommandBuffer,Memory);\ + gcmASSERT((gctUINT32)Count <= 1024); \ + \ + *Memory++ \ + = gcmSETFIELDVALUE(0, AQ_COMMAND_LOAD_STATE_COMMAND, OPCODE, LOAD_STATE) \ + | gcmSETFIELD (0, AQ_COMMAND_LOAD_STATE_COMMAND, FLOAT, FixedPoint) \ + | gcmSETFIELD (0, AQ_COMMAND_LOAD_STATE_COMMAND, COUNT, Count) \ + | gcmSETFIELD (0, AQ_COMMAND_LOAD_STATE_COMMAND, ADDRESS, Address); \ +} + +#define gcmENDSTATEBATCH_NEW(CommandBuffer, Memory) \ + gcmVERIFYLOADSTATEALIGNED(CommandBuffer,Memory); + +/*----------------------------------------------------------------------------*/ + +#define gcmSETSTATEDATA_NEW(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + gcmSAFECASTSIZET(__temp_data32__, Data); \ + \ + *Memory++ = __temp_data32__; \ + \ + gcoHARDWARE_UpdateDelta(\ + StateDelta, Address, 0, __temp_data32__ \ + ); \ + \ + gcmDUMPSTATEDATA(StateDelta, FixedPoint, Address, __temp_data32__); \ +} + +#define gcmSETSTATEDATAWITHMASK_NEW(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + __temp_data32__ = Data; \ + \ + *Memory++ = __temp_data32__; \ + \ + gcoHARDWARE_UpdateDelta(\ + StateDelta, Address, Mask, __temp_data32__ \ + ); \ + \ + gcmDUMPSTATEDATA(StateDelta, FixedPoint, Address, __temp_data32__); \ +} + + +#define gcmSETCTRLSTATE_NEW(StateDelta, CommandBuffer, Memory, Address, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + __temp_data32__ = Data; \ + \ + *Memory++ = __temp_data32__; \ + \ + gcmDUMPSTATEDATA(StateDelta, gcvFALSE, Address, __temp_data32__); \ +} + +#define gcmSETFILLER_NEW(CommandBuffer, Memory) \ +{ \ + *(gctUINT32_PTR)Memory = 0x18000000; \ + Memory += 1; \ +} + +/*----------------------------------------------------------------------------*/ + +#define gcmSETSINGLESTATE_DUMMY(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ + { \ + gctUINT32 __temp_data32__; \ + __temp_data32__ = Data ; \ + gcmVERIFYLOADSTATEALIGNED(CommandBuffer, Memory); \ + *Memory++ = \ + (gctUINT32)(0) | (0xFFFF & Address); \ + *Memory++ = __temp_data32__; \ + gcmENDSTATEBATCH_NEW(CommandBuffer, Memory); \ +} + +#define gcmSETSINGLESTATE_NEW(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gcmBEGINSTATEBATCH_NEW(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETSTATEDATA_NEW(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data); \ + gcmENDSTATEBATCH_NEW(CommandBuffer, Memory); \ +} + +#define gcmSETSINGLESTATEWITHMASK_NEW(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data) \ +{ \ + gcmBEGINSTATEBATCH_NEW(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETSTATEDATAWITHMASK_NEW(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data); \ + gcmENDSTATEBATCH_NEW(CommandBuffer, Memory); \ +} + + +#define gcmSETSINGLECTRLSTATE_NEW(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gcmBEGINSTATEBATCH_NEW(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETCTRLSTATE_NEW(StateDelta, CommandBuffer, Memory, Address, Data); \ + gcmENDSTATEBATCH_NEW(CommandBuffer, Memory); \ +} + + + +#define gcmSETSEMASTALLPIPE_NEW(StateDelta, CommandBuffer, Memory, Data) \ +{ \ + gcmSETSINGLECTRLSTATE_NEW(StateDelta, CommandBuffer, Memory, gcvFALSE, 0x0E02, Data); \ + \ + *Memory++ = gcmSETFIELDVALUE(0, STALL_COMMAND, OPCODE, STALL); \ + \ + *Memory++ = Data; \ + \ + gcmDUMP(gcvNULL, "#[stall 0x%08X 0x%08X]", \ + gcmSETFIELDVALUE(0, AQ_SEMAPHORE, SOURCE, FRONT_END), \ + gcmSETFIELDVALUE(0, AQ_SEMAPHORE, DESTINATION, PIXEL_ENGINE)); \ +} + +#define gcmSETSTARTDECOMMAND_NEW(CommandBuffer, Memory, Count) \ +{ \ + *Memory++ \ + = gcmSETFIELDVALUE(0, AQ_COMMAND_START_DE_COMMAND, OPCODE, START_DE) \ + | gcmSETFIELD (0, AQ_COMMAND_START_DE_COMMAND, COUNT, Count) \ + | gcmSETFIELD (0, AQ_COMMAND_START_DE_COMMAND, DATA_COUNT, 0); \ + \ + *Memory++ = 0xDEADDEED; \ + \ +} + +#define gcmSETSTATEDATA_NEW_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + __temp_data32__ = Data; \ + \ + *Memory++ = __temp_data32__; \ + \ + gcmDUMPSTATEDATA(StateDelta, FixedPoint, Address, __temp_data32__); \ +} + +#define gcmSETSTATEDATAWITHMASK_NEW_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + __temp_data32__ = Data; \ + \ + *Memory++ = __temp_data32__; \ + \ + gcmDUMPSTATEDATA(StateDelta, FixedPoint, Address, __temp_data32__); \ +} + +#define gcmSETSINGLESTATE_NEW_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gcmBEGINSTATEBATCH_NEW(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETSTATEDATA_NEW_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data); \ + gcmENDSTATEBATCH_NEW(CommandBuffer, Memory); \ +} + +#define gcmSETSINGLESTATEWITHMASK_NEW_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data) \ +{ \ + gcmBEGINSTATEBATCH_NEW(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETSTATEDATAWITHMASK_NEW_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data); \ + gcmENDSTATEBATCH_NEW(CommandBuffer, Memory); \ +} + +#define gcmSETSTATEDATA_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + gcmVERIFYLOADSTATE(CommandBuffer, Memory, Address); \ + \ + gcmSAFECASTSIZET(__temp_data32__, Data); \ + \ + *Memory++ = __temp_data32__; \ + \ + gcmDUMPSTATEDATA(StateDelta, FixedPoint, Address, __temp_data32__); \ +} + +#define gcmSETSTATEDATAWITHMASK_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data) \ +{ \ + gctUINT32 __temp_data32__; \ + \ + gcmVERIFYLOADSTATE(CommandBuffer, Memory, Address); \ + \ + __temp_data32__ = Data; \ + \ + *Memory++ = __temp_data32__; \ + \ + gcmDUMPSTATEDATA(StateDelta, FixedPoint, Address, __temp_data32__); \ +} + +#define gcmSETSINGLESTATE_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data) \ +{ \ + gcmBEGINSTATEBATCH(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETSTATEDATA_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Data); \ + gcmENDSTATEBATCH(CommandBuffer, Memory); \ +} + +#define gcmSETSINGLESTATEWITHMASK_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data) \ +{ \ + gcmBEGINSTATEBATCH(CommandBuffer, Memory, FixedPoint, Address, 1); \ + gcmSETSTATEDATAWITHMASK_FAST(StateDelta, CommandBuffer, Memory, FixedPoint, \ + Address, Mask, Data); \ + gcmENDSTATEBATCH(CommandBuffer, Memory); \ +} + +#define gcmDEFINESTATEBUFFER_NEW_FAST(CommandBuffer, Memory) \ + gcmDEFINELOADSTATEBASE() \ + gcsTEMPCMDBUF CommandBuffer = gcvNULL; \ + gctUINT32_PTR Memory; + +#define gcmDEFINESTATEBUFFER_FAST(CommandBuffer, Memory, ReserveSize) \ + gctSIZE_T ReserveSize; \ + gcoCMDBUF CommandBuffer; \ + gctUINT32_PTR Memory; + +#define gcmBEGINSTATEBUFFER_FAST(Hardware, CommandBuffer, Memory, ReserveSize) \ +{ \ + gcmONERROR(gcoBUFFER_Reserve(\ + Hardware->engine[gcvENGINE_RENDER].buffer, ReserveSize, gcvTRUE, &CommandBuffer \ + )); \ + \ + Memory = (gctUINT32_PTR) gcmUINT64_TO_PTR(CommandBuffer->lastReserve); \ + \ +} + +#define gcmBEGINSTATEBUFFER_NEW_FAST(Hardware, CommandBuffer, Memory, OutSide) \ +{ \ + if (OutSide) \ + {\ + Memory = (gctUINT32_PTR)*OutSide; \ + }\ + else \ + {\ + gcmONERROR(gcoBUFFER_StartTEMPCMDBUF(\ + Hardware->engine[gcvENGINE_RENDER].buffer, Hardware->engine[gcvENGINE_RENDER].queue, &CommandBuffer \ + ));\ + \ + Memory = (gctUINT32_PTR)(CommandBuffer->buffer); \ + \ + }\ + \ + gcmSETLOADSTATEBASE(CommandBuffer,OutSide);\ +} + +#define gcmENDSTATEBUFFER_NEW_FAST(Hardware, CommandBuffer, Memory, OutSide) \ +{ \ + if (OutSide) \ + {\ + *OutSide = Memory; \ + }\ + else \ + {\ + CommandBuffer->currentByteSize = (gctUINT32)((gctUINT8_PTR)Memory - \ + (gctUINT8_PTR)CommandBuffer->buffer); \ + \ + gcmONERROR(gcoBUFFER_EndTEMPCMDBUF(Hardware->engine[gcvENGINE_RENDER].buffer, gcvFALSE));\ + }\ + gcmUNSETLOADSTATEBASE()\ +} + +/******************************************************************************* +** +** gcmCONFIGUREUNIFORMS +** +** Configure uniforms according to chip and numConstants. +*/ +#if !gcdENABLE_UNIFIED_CONSTANT +#define gcmCONFIGUREUNIFORMS(ChipModel, ChipRevision, NumConstants, \ + UnifiedConst, VsConstBase, PsConstBase, VsConstMax, PsConstMax, ConstMax) \ +{ \ + if (ChipModel == gcv2000 && (ChipRevision == 0x5118 || ChipRevision == 0x5140)) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstBase = 0x1400; \ + PsConstBase = 0x1C00; \ + VsConstMax = 256; \ + PsConstMax = 64; \ + ConstMax = 320; \ + } \ + else if (NumConstants == 320) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstBase = 0x1400; \ + PsConstBase = 0x1C00; \ + VsConstMax = 256; \ + PsConstMax = 64; \ + ConstMax = 320; \ + } \ + /* All GC1000 series chips can only support 64 uniforms for ps on non-unified const mode. */ \ + else if (NumConstants > 256 && ChipModel == gcv1000) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstBase = 0x1400; \ + PsConstBase = 0x1C00; \ + VsConstMax = 256; \ + PsConstMax = 64; \ + ConstMax = 320; \ + } \ + else if (NumConstants > 256) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstBase = 0x1400; \ + PsConstBase = 0x1C00; \ + VsConstMax = 256; \ + PsConstMax = 256; \ + ConstMax = 512; \ + } \ + else if (NumConstants == 256) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstBase = 0x1400; \ + PsConstBase = 0x1C00; \ + VsConstMax = 256; \ + PsConstMax = 256; \ + ConstMax = 512; \ + } \ + else \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstBase = 0x1400; \ + PsConstBase = 0x1C00; \ + VsConstMax = 168; \ + PsConstMax = 64; \ + ConstMax = 232; \ + } \ +} +#else +#define gcmCONFIGUREUNIFORMS(ChipModel, ChipRevision, Halti5Avail, SmallBatch, NumConstants, \ + UnifiedConst, VsConstBase, PsConstBase, VsConstMax, PsConstMax, ConstMax) \ +{ \ + if (NumConstants > 256) \ + { \ + UnifiedConst = gcvTRUE; \ + if (SmallBatch) \ + { \ + VsConstBase = 0xD000; \ + PsConstBase = 0xD000; \ + } \ + else if (Halti5Avail) \ + { \ + VsConstBase = 0xD000; \ + PsConstBase = 0xD800; \ + } \ + else \ + {\ + VsConstBase = 0xC000; \ + PsConstBase = 0xC000; \ + }\ + if ((ChipModel == gcv880) && ((ChipRevision & 0xfff0) == 0x5120)) \ + { \ + VsConstMax = 512; \ + PsConstMax = 64; \ + ConstMax = 576; \ + } \ + else \ + { \ + VsConstMax = gcmMIN(512, NumConstants - 64); \ + PsConstMax = gcmMIN(512, NumConstants - 64); \ + ConstMax = NumConstants; \ + } \ + } \ + else if (NumConstants == 256) \ + { \ + if (ChipModel == gcv2000 && (ChipRevision == 0x5118 || ChipRevision == 0x5140)) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstBase = 0x1400; \ + PsConstBase = 0x1C00; \ + VsConstMax = 256; \ + PsConstMax = 64; \ + ConstMax = 320; \ + } \ + else \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstBase = 0x1400; \ + PsConstBase = 0x1C00; \ + VsConstMax = 256; \ + PsConstMax = 256; \ + ConstMax = 512; \ + } \ + } \ + else \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstBase = 0x1400; \ + PsConstBase = 0x1C00; \ + VsConstMax = 168; \ + PsConstMax = 64; \ + ConstMax = 232; \ + } \ +} +#endif + +/******************************************************************************* +** +** gcmCONFIGUREUNIFORMS2 +** only fix clang build error +** +** Configure uniforms according to chip and numConstants. +*/ +#if !gcdENABLE_UNIFIED_CONSTANT +#define gcmCONFIGUREUNIFORMS2(ChipModel, ChipRevision, NumConstants, \ + UnifiedConst, VsConstMax, PsConstMax) \ +{ \ + if (ChipModel == gcv2000 && (ChipRevision == 0x5118 || ChipRevision == 0x5140)) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstMax = 256; \ + PsConstMax = 64; \ + } \ + else if (NumConstants == 320) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstMax = 256; \ + PsConstMax = 64; \ + } \ + /* All GC1000 series chips can only support 64 uniforms for ps on non-unified const mode. */ \ + else if (NumConstants > 256 && ChipModel == gcv1000) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstMax = 256; \ + PsConstMax = 64; \ + } \ + else if (NumConstants > 256) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstMax = 256; \ + PsConstMax = 256; \ + } \ + else if (NumConstants == 256) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstMax = 256; \ + PsConstMax = 256; \ + } \ + else \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstMax = 168; \ + PsConstMax = 64; \ + } \ +} +#else +#define gcmCONFIGUREUNIFORMS2(ChipModel, ChipRevision, Halti5Avail, SmallBatch, NumConstants, \ + UnifiedConst, VsConstMax, PsConstMax) \ +{ \ + if (NumConstants > 256) \ + { \ + UnifiedConst = gcvTRUE; \ + if ((ChipModel == gcv880) && ((ChipRevision & 0xfff0) == 0x5120)) \ + { \ + VsConstMax = 512; \ + PsConstMax = 64; \ + } \ + else \ + { \ + VsConstMax = gcmMIN(512, NumConstants - 64); \ + PsConstMax = gcmMIN(512, NumConstants - 64); \ + } \ + } \ + else if (NumConstants == 256) \ + { \ + if (ChipModel == gcv2000 && (ChipRevision == 0x5118 || ChipRevision == 0x5140)) \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstMax = 256; \ + PsConstMax = 64; \ + } \ + else \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstMax = 256; \ + PsConstMax = 256; \ + } \ + } \ + else \ + { \ + UnifiedConst = gcvFALSE; \ + VsConstMax = 168; \ + PsConstMax = 64; \ + } \ +} +#endif + +#define gcmAnyTileStatusEnableForFullMultiSlice(SurfView, anyTsEnableForMultiSlice)\ +{\ + gctUINT i = 0; \ + for (; i < (SurfView->surf->requestD); i++)\ + {\ + if ((SurfView->surf->tileStatusNode.pool != gcvPOOL_UNKNOWN) && \ + (SurfView->surf->tileStatusDisabled[i] == gcvFALSE))\ + {\ + *anyTsEnableForMultiSlice = gcvTRUE;\ + break;\ + }\ + }\ +}\ + +#define gcmAnyTileStatusEnableForMultiSlice(SurfView, anyTsEnableForMultiSlice)\ +{\ + gctUINT i = SurfView->firstSlice; \ + for (; i < (SurfView->firstSlice + SurfView->numSlices); i++)\ + {\ + if ((SurfView->surf->tileStatusNode.pool != gcvPOOL_UNKNOWN) && \ + (SurfView->surf->tileStatusDisabled[i] == gcvFALSE))\ + {\ + *anyTsEnableForMultiSlice = gcvTRUE;\ + break;\ + }\ + }\ +}\ + +#define gcmCanTileStatusEnabledForMultiSlice(SurfView, canTsEnabled)\ +{\ + if (SurfView->numSlices > 1)\ + {\ + if (SurfView->surf->tileStatusNode.pool != gcvPOOL_UNKNOWN) \ + {\ + gctUINT i = 0;\ + for (; i < SurfView->numSlices; i++)\ + {\ + if (SurfView->surf->tileStatusDisabled[i] == gcvTRUE)\ + {\ + *canTsEnabled = gcvFALSE;\ + break;\ + }\ + if (SurfView->surf->fcValue[i] != SurfView->surf->fcValue[0])\ + {\ + *canTsEnabled = gcvFALSE;\ + break;\ + }\ + \ + if (SurfView->surf->fcValueUpper[i] != SurfView->surf->fcValueUpper[0])\ + {\ + *canTsEnabled = gcvFALSE;\ + break;\ + }\ + }\ + }\ + else\ + {\ + *canTsEnabled = gcvFALSE;\ + }\ + }\ + else\ + {\ + if ((SurfView->surf->tileStatusNode.pool == gcvPOOL_UNKNOWN) || (SurfView->surf->tileStatusDisabled[SurfView->firstSlice] == gcvTRUE))\ + {\ + *canTsEnabled = gcvFALSE;\ + }\ + }\ +}\ + + +#define gcmCONFIGUSC(prefix, featureUSC, featureSeparateLS, featureComputeOnly, \ + featureTS, featureGS, featureUSCFullCacheFix, featureL1CacheSize, featureUSCMaxPages, \ + attribCacheRatio, L1CacheRatio) \ +{ \ + attribCacheRatio = 0x2; \ + \ + if (featureUSC) \ + { \ + if (featureSeparateLS) \ + { \ + L1CacheRatio = 0x0; \ + } \ + else \ + { \ + gctUINT L1cacheSize; \ + \ + if (featureComputeOnly) \ + { \ + L1cacheSize = featureL1CacheSize; \ + } \ + else \ + { \ + gctUINT attribBufSizeInKB; \ + if (featureTS) \ + { \ + /* GS/TS must be bundled. */ \ + prefix##ASSERT(featureGS); \ + featureGS = featureGS; \ + attribBufSizeInKB = 42; \ + } \ + else \ + { \ + prefix##ASSERT(!featureGS); \ + attribBufSizeInKB = 8; \ + } \ + if (attribBufSizeInKB < featureUSCMaxPages) \ + { \ + L1cacheSize = featureUSCMaxPages - attribBufSizeInKB; \ + } \ + else \ + { \ + attribBufSizeInKB -= 2; \ + L1cacheSize = 2; \ + } \ + } \ + prefix##ASSERT(L1cacheSize); \ + if (L1cacheSize >= featureL1CacheSize) \ + { \ + L1CacheRatio = 0x0; \ + prefix##ASSERT(featureUSCFullCacheFix); \ + featureUSCFullCacheFix = featureUSCFullCacheFix; \ + } \ + else \ + { \ + static const gctINT s_uscCacheRatio[] = \ + { \ + 100000,/* 1.0f */ \ + 50000, /* 0.5f */ \ + 25000, /* 0.25f */ \ + 12500, /* 0.125f */ \ + 62500, /* 0.0625f */ \ + 3125, /* 0.03125f */ \ + 75000, /* 0.75f */ \ + 0, /*0.0f */ \ + }; \ + gctINT maxL1cacheSize = L1cacheSize * 100000; \ + gctINT delta = 2147483647; /* start with very big delta */ \ + gctINT i = 0; \ + gctINT curIndex = -1; \ + for (; i < gcmCOUNTOF(s_uscCacheRatio); ++i) \ + { \ + gctINT curL1cacheSize = featureL1CacheSize * s_uscCacheRatio[i]; \ + \ + if ((maxL1cacheSize >= curL1cacheSize) && \ + ((maxL1cacheSize - curL1cacheSize) < delta)) \ + { \ + curIndex = i; \ + delta = maxL1cacheSize - curL1cacheSize; \ + } \ + } \ + prefix##ASSERT(-1 != curIndex); \ + L1CacheRatio = curIndex; \ + } \ + } \ + } \ +} \ + +#define gcmCONFIGUSC2(prefix, featureUSC, featureSeparateLS, featureComputeOnly, \ + featureTS, featureL1CacheSize, featureUSCMaxPages, \ + attribCacheRatio, L1CacheRatio) \ +{ \ + attribCacheRatio = 0x2; \ + \ + if (featureUSC) \ + { \ + if (featureSeparateLS) \ + { \ + L1CacheRatio = 0x0; \ + } \ + else \ + { \ + gctUINT L1cacheSize; \ + \ + if (featureComputeOnly) \ + { \ + L1cacheSize = featureL1CacheSize; \ + } \ + else \ + { \ + gctUINT attribBufSizeInKB; \ + if (featureTS) \ + { \ + /* GS/TS must be bundled. */ \ + attribBufSizeInKB = 42; \ + } \ + else \ + { \ + attribBufSizeInKB = 8; \ + } \ + if (attribBufSizeInKB < featureUSCMaxPages) \ + { \ + L1cacheSize = featureUSCMaxPages - attribBufSizeInKB; \ + } \ + else \ + { \ + attribBufSizeInKB -= 2; \ + L1cacheSize = 2; \ + } \ + } \ + prefix##ASSERT(L1cacheSize); \ + if (L1cacheSize >= featureL1CacheSize) \ + { \ + L1CacheRatio = 0x0; \ + } \ + else \ + { \ + static const gctINT s_uscCacheRatio[] = \ + { \ + 100000,/* 1.0f */ \ + 50000, /* 0.5f */ \ + 25000, /* 0.25f */ \ + 12500, /* 0.125f */ \ + 62500, /* 0.0625f */ \ + 3125, /* 0.03125f */ \ + 75000, /* 0.75f */ \ + 0, /*0.0f */ \ + }; \ + gctINT maxL1cacheSize = L1cacheSize * 100000; \ + gctINT delta = 2147483647; /* start with very big delta */ \ + gctINT i = 0; \ + gctINT curIndex = -1; \ + for (; i < gcmCOUNTOF(s_uscCacheRatio); ++i) \ + { \ + gctINT curL1cacheSize = featureL1CacheSize * s_uscCacheRatio[i]; \ + \ + if ((maxL1cacheSize >= curL1cacheSize) && \ + ((maxL1cacheSize - curL1cacheSize) < delta)) \ + { \ + curIndex = i; \ + delta = maxL1cacheSize - curL1cacheSize; \ + } \ + } \ + prefix##ASSERT(-1 != curIndex); \ + L1CacheRatio = curIndex; \ + } \ + } \ + } \ +} \ + +#if VIVANTE_PROFILER_SYSTEM_MEMORY +typedef struct _memory_profile_info +{ + struct + { + gctUINT64 currentSize; + gctUINT64 peakSize; + gctUINT64 total_allocate; + gctUINT64 total_free; + gctUINT32 total_allocateCount; + gctUINT32 total_freeCount; + } system_memory, gpu_memory; +} memory_profile_info; + +gceSTATUS +gcoOS_GetMemoryProfileInfo( + size_t size, + struct _memory_profile_info *info + ); + +gceSTATUS gcoOS_DumpMemoryProfile(void); +gceSTATUS gcoOS_InitMemoryProfile(void); +gceSTATUS gcoOS_DeInitMemoryProfile(void); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __gc_hal_base_h_ */ + +
diff --git a/hal/kernel/inc/gc_hal_debug_zones.h b/hal/kernel/inc/gc_hal_debug_zones.h new file mode 100644 index 0000000..6d56d0c --- /dev/null +++ b/hal/kernel/inc/gc_hal_debug_zones.h
@@ -0,0 +1,329 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_debug_zones_h_ +#define __gc_hal_debug_zones_h_ + +#ifdef __cplusplus +extern "C" { +#endif + +/******************************************************************************\ + ************************ Debug Zone Pattern Summary *************************** + * A debug zone is an unsigned integer of 32 bit (Bit 31- Bit 0). * + * Bit 31 to 28 defines API, which is 0 for HAL API and has value of 1 - 14 * + * for Khronos API. Value 15 (0xF) is reserved for gcdZONE_NONE. * + * Bit 27 to 0 defines subzones of each API. Value 0xFFFFFFF is resevered for * + * gcdZONE_ALL. * + * * +\******************************************************************************/ + +/* Retrieve API bits 31 to 28 */ +#define gcmZONE_GET_API(zone) ((zone) >> 28) + +/* Retrieve Subzone bits 27 to 0 */ +#define gcmZONE_GET_SUBZONES(zone) ((zone) << 4) + +/******************************************************************************\ +******************************** HAL Zone ************************************** +\******************************************************************************/ + +#define gcdZONE_API_HAL ((gctUINT32) 0 << 28) + +/******************************************************************************\ +******************************** HAL Subzones ********************************** +\******************************************************************************/ + +/* Subzones Kernel and User have in common */ +#define gcvZONE_OS (1 << 0) +#define gcvZONE_HARDWARE (1 << 1) +#define gcvZONE_HEAP (1 << 2) +#define gcvZONE_SIGNAL (1 << 3) + +/* Subzones of HAL Kernel */ +#define gcvZONE_KERNEL (1 << 4) +#define gcvZONE_VIDMEM (1 << 5) +#define gcvZONE_COMMAND (1 << 6) +#define gcvZONE_DRIVER (1 << 7) +#define gcvZONE_CMODEL (1 << 8) +#define gcvZONE_MMU (1 << 9) +#define gcvZONE_EVENT (1 << 10) +#define gcvZONE_DEVICE (1 << 11) +#define gcvZONE_DATABASE (1 << 12) +#define gcvZONE_INTERRUPT (1 << 13) +#define gcvZONE_POWER (1 << 14) +#define gcvZONE_ASYNC_COMMAND (1 << 15) +#define gcvZONE_ALLOCATOR (1 << 16) + +/* Subzones of HAL User */ +#define gcdZONE_HAL_API (1 << 4) +#define gcdZONE_BUFFER (1 << 5) +#define gcdZONE_VGBUFFER (1 << 6) +#define gcdZONE_SURFACE (1 << 7) +#define gcdZONE_INDEX (1 << 8) +#define gcdZONE_STREAM (1 << 9) +#define gcdZONE_TEXTURE (1 << 10) +#define gcdZONE_2D (1 << 11) +#define gcdZONE_3D (1 << 12) +#define gcdZONE_COMPILER (1 << 13) +#define gcdZONE_MEM (1 << 14) +#define gcdZONE_VERTEXARRAY (1 << 15) +#define gcdZONE_CL (1 << 16) +#define gcdZONE_VG (1 << 17) +#define gcdZONE_VX (1 << 18) +#define gcdZONE_UTILITY (1 << 19) +#define gcdZONE_RECT (1 << 20) +#define gcdZONE_BUFOBJ (1 << 21) +#define gcdZONE_PROFILER (1 << 22) +#define gcdZONE_SHADER (1 << 23) + + +/******************************************************************************\ +******************************** Khronos API Zones ***************************** +\******************************************************************************/ + +#define gcdZONE_API_EGL ((gctUINT32) 1 << 28) +#define gcdZONE_API_ES11 ((gctUINT32) 2 << 28) +#define gcdZONE_API_ES30 ((gctUINT32) 3 << 28) +#define gcdZONE_API_GL40 ((gctUINT32) 4 << 28) +#define gcdZONE_API_VG3D ((gctUINT32) 5 << 28) +#define gcdZONE_API_CL ((gctUINT32) 6 << 28) +#define gcdZONE_API_VX ((gctUINT32) 7 << 28) +#define gcdZONE_API_VG ((gctUINT32) 8 << 28) + +/******************************************************************************\ +************************* Subzones of Khronos API Zones ************************ +\******************************************************************************/ + +/* Subzones of EGL API */ +#define gcdZONE_EGL_API (gcdZONE_API_EGL | (1 << 0)) +#define gcdZONE_EGL_SURFACE (gcdZONE_API_EGL | (1 << 1)) +#define gcdZONE_EGL_CONTEXT (gcdZONE_API_EGL | (1 << 2)) +#define gcdZONE_EGL_CONFIG (gcdZONE_API_EGL | (1 << 3)) +#define gcdZONE_EGL_OS (gcdZONE_API_EGL | (1 << 4)) /* unused */ +#define gcdZONE_EGL_IMAGE (gcdZONE_API_EGL | (1 << 5)) +#define gcdZONE_EGL_SWAP (gcdZONE_API_EGL | (1 << 6)) +#define gcdZONE_EGL_INIT (gcdZONE_API_EGL | (1 << 7)) +#define gcdZONE_EGL_SYNC (gcdZONE_API_EGL | (1 << 8)) +#define gcdZONE_EGL_COMPOSE (gcdZONE_API_EGL | (1 << 9)) /* unused */ +#define gcdZONE_EGL_RENDER_THREAD (gcdZONE_API_EGL | (1 << 10)) /* unused */ + +/* Subzones of ES11 API */ +#define gcdZONE_ES11_BUFFER (gcdZONE_API_ES11 | (1 << 0)) +#define gcdZONE_ES11_CLEAR (gcdZONE_API_ES11 | (1 << 1)) +#define gcdZONE_ES11_CLIP (gcdZONE_API_ES11 | (1 << 2)) +#define gcdZONE_ES11_CONTEXT (gcdZONE_API_ES11 | (1 << 3)) +#define gcdZONE_ES11_DRAW (gcdZONE_API_ES11 | (1 << 4)) +#define gcdZONE_ES11_ENABLE (gcdZONE_API_ES11 | (1 << 5)) +#define gcdZONE_ES11_EXTENTION (gcdZONE_API_ES11 | (1 << 6)) +#define gcdZONE_ES11_FOG (gcdZONE_API_ES11 | (1 << 7)) +#define gcdZONE_ES11_FRAGMENT (gcdZONE_API_ES11 | (1 << 8)) +#define gcdZONE_ES11_LIGHT (gcdZONE_API_ES11 | (1 << 9)) +#define gcdZONE_ES11_MATRIX (gcdZONE_API_ES11 | (1 << 10)) +#define gcdZONE_ES11_PIXEL (gcdZONE_API_ES11 | (1 << 11)) +#define gcdZONE_ES11_POLIGON (gcdZONE_API_ES11 | (1 << 12)) +#define gcdZONE_ES11_LINE (gcdZONE_API_ES11 | (1 << 13)) /* unused */ +#define gcdZONE_ES11_QUERY (gcdZONE_API_ES11 | (1 << 14)) +#define gcdZONE_ES11_TEXTURE (gcdZONE_API_ES11 | (1 << 15)) +#define gcdZONE_ES11_STATES (gcdZONE_API_ES11 | (1 << 16)) +#define gcdZONE_ES11_STREAM (gcdZONE_API_ES11 | (1 << 17)) +#define gcdZONE_ES11_VIEWPORT (gcdZONE_API_ES11 | (1 << 18)) +#define gcdZONE_ES11_SHADER (gcdZONE_API_ES11 | (1 << 19)) +#define gcdZONE_ES11_HASH (gcdZONE_API_ES11 | (1 << 20)) +#define gcdZONE_ES11_TRACE (gcdZONE_API_ES11 | (1 << 21)) + +/* Subzones of ES30 API */ +#define gcdZONE_ES30_TRACE (gcdZONE_API_ES30 | (1 << 0)) +#define gcdZONE_ES30_BUFFER (gcdZONE_API_ES30 | (1 << 1)) +#define gcdZONE_ES30_CLEAR (gcdZONE_API_ES30 | (1 << 2)) +#define gcdZONE_ES30_CODEC (gcdZONE_API_ES30 | (1 << 3)) +#define gcdZONE_ES30_CONTEXT (gcdZONE_API_ES30 | (1 << 4)) +#define gcdZONE_ES30_DEPTH (gcdZONE_API_ES30 | (1 << 5)) +#define gcdZONE_ES30_DEVICE (gcdZONE_API_ES30 | (1 << 6)) +#define gcdZONE_ES30_DRAW (gcdZONE_API_ES30 | (1 << 7)) +#define gcdZONE_ES30_FBO (gcdZONE_API_ES30 | (1 << 8)) +#define gcdZONE_ES30_PIXEL (gcdZONE_API_ES30 | (1 << 9)) +#define gcdZONE_ES30_SHADER (gcdZONE_API_ES30 | (1 << 10)) +#define gcdZONE_ES30_STATE (gcdZONE_API_ES30 | (1 << 11)) +#define gcdZONE_ES30_TEXTURE (gcdZONE_API_ES30 | (1 << 12)) +#define gcdZONE_ES30_UTILS (gcdZONE_API_ES30 | (1 << 13)) +#define gcdZONE_ES30_PROFILER (gcdZONE_API_ES30 | (1 << 14)) +#define gcdZONE_ES30_CORE (gcdZONE_API_ES30 | (1 << 15)) + +/* Subzones of GL40 API */ +#define gcdZONE_GL40_TRACE (gcdZONE_API_GL40 | (1 << 0)) +#define gcdZONE_GL40_BUFFER (gcdZONE_API_GL40 | (1 << 1)) +#define gcdZONE_GL40_CLEAR (gcdZONE_API_GL40 | (1 << 2)) /* unused */ +#define gcdZONE_GL40_CODEC (gcdZONE_API_GL40 | (1 << 3)) +#define gcdZONE_GL40_CONTEXT (gcdZONE_API_GL40 | (1 << 4)) +#define gcdZONE_GL40_DEPTH (gcdZONE_API_GL40 | (1 << 5)) +#define gcdZONE_GL40_DEVICE (gcdZONE_API_GL40 | (1 << 6)) +#define gcdZONE_GL40_DRAW (gcdZONE_API_GL40 | (1 << 7)) +#define gcdZONE_GL40_FBO (gcdZONE_API_GL40 | (1 << 8)) +#define gcdZONE_GL40_PIXEL (gcdZONE_API_GL40 | (1 << 9)) +#define gcdZONE_GL40_SHADER (gcdZONE_API_GL40 | (1 << 10)) +#define gcdZONE_GL40_STATE (gcdZONE_API_GL40 | (1 << 11)) +#define gcdZONE_GL40_TEXTURE (gcdZONE_API_GL40 | (1 << 12)) +#define gcdZONE_GL40_UTILS (gcdZONE_API_GL40 | (1 << 13)) +#define gcdZONE_GL40_PROFILER (gcdZONE_API_GL40 | (1 << 14)) +#define gcdZONE_GL40_CORE (gcdZONE_API_GL40 | (1 << 15)) +#define gcdZONE_GL40_FIXVERTEX (gcdZONE_API_GL40 | (1 << 16)) +#define gcdZONE_GL40_FIXFRAG (gcdZONE_API_GL40 | (1 << 17)) +#define gcdZONE_GL40_HASH (gcdZONE_API_GL40 | (1 << 18)) + +/* Subzones of VG3D API */ +#define gcdZONE_VG3D_CONTEXT (gcdZONE_API_VG3D | (1 << 0)) +#define gcdZONE_VG3D_DUMP (gcdZONE_API_VG3D | (1 << 1)) +#define gcdZONE_VG3D_EGL (gcdZONE_API_VG3D | (1 << 2)) +#define gcdZONE_VG3D_FONT (gcdZONE_API_VG3D | (1 << 3)) +#define gcdZONE_VG3D_HARDWARE (gcdZONE_API_VG3D | (1 << 4)) +#define gcdZONE_VG3D_IMAGE (gcdZONE_API_VG3D | (1 << 5)) +#define gcdZONE_VG3D_MASK (gcdZONE_API_VG3D | (1 << 6)) +#define gcdZONE_VG3D_MATRIX (gcdZONE_API_VG3D | (1 << 7)) +#define gcdZONE_VG3D_OBJECT (gcdZONE_API_VG3D | (1 << 8)) +#define gcdZONE_VG3D_PAINT (gcdZONE_API_VG3D | (1 << 9)) +#define gcdZONE_VG3D_PATH (gcdZONE_API_VG3D | (1 << 10)) +#define gcdZONE_VG3D_PROFILER (gcdZONE_API_VG3D | (1 << 11)) +#define gcdZONE_VG3D_SCANLINE (gcdZONE_API_VG3D | (1 << 12)) +#define gcdZONE_VG3D_SHADER (gcdZONE_API_VG3D | (1 << 13)) +#define gcdZONE_VG3D_TESSELLATOR (gcdZONE_API_VG3D | (1 << 14)) +#define gcdZONE_VG3D_VGU (gcdZONE_API_VG3D | (1 << 15)) + +/* Subzones of VG11 API */ +#define gcdZONE_VG_ARC (gcdZONE_API_VG | (1 << 0)) +#define gcdZONE_VG_CONTEXT (gcdZONE_API_VG | (1 << 1)) +#define gcdZONE_VG_DEBUG (gcdZONE_API_VG | (1 << 2)) +#define gcdZONE_VG_FILTER (gcdZONE_API_VG | (1 << 3)) +#define gcdZONE_VG_FORMAT (gcdZONE_API_VG | (1 << 4)) +#define gcdZONE_VG_IMAGE (gcdZONE_API_VG | (1 << 5)) +#define gcdZONE_VG_MAIN (gcdZONE_API_VG | (1 << 6)) +#define gcdZONE_VG_MASK (gcdZONE_API_VG | (1 << 7)) +#define gcdZONE_VG_MATRIX (gcdZONE_API_VG | (1 << 8)) +#define gcdZONE_VG_MEMORYMGR (gcdZONE_API_VG | (1 << 9)) +#define gcdZONE_VG_OBJECT (gcdZONE_API_VG | (1 << 10)) +#define gcdZONE_VG_PAINT (gcdZONE_API_VG | (1 << 11)) +#define gcdZONE_VG_PATH (gcdZONE_API_VG | (1 << 12)) +#define gcdZONE_VG_STATE (gcdZONE_API_VG | (1 << 13)) +#define gcdZONE_VG_STROKE (gcdZONE_API_VG | (1 << 14)) +#define gcdZONE_VG_TEXT (gcdZONE_API_VG | (1 << 15)) +#define gcdZONE_VG_VGU (gcdZONE_API_VG | (1 << 16)) + +/* Subzones of CL API */ +#define gcdZONE_CL_COMMAND (gcdZONE_API_CL | (1 << 0)) +#define gcdZONE_CL_CONTEXT (gcdZONE_API_CL | (1 << 1)) +#define gcdZONE_CL_DEVICE (gcdZONE_API_CL | (1 << 2)) +#define gcdZONE_CL_ENQUEUE (gcdZONE_API_CL | (1 << 3)) +#define gcdZONE_CL_EVENT (gcdZONE_API_CL | (1 << 4)) +#define gcdZONE_CL_EXT (gcdZONE_API_CL | (1 << 5)) +#define gcdZONE_CL_GL (gcdZONE_API_CL | (1 << 6)) +#define gcdZONE_CL_KERNEL (gcdZONE_API_CL | (1 << 7)) +#define gcdZONE_CL_MEM (gcdZONE_API_CL | (1 << 8)) +#define gcdZONE_CL_PLATFORM (gcdZONE_API_CL | (1 << 9)) +#define gcdZONE_CL_PROFILER (gcdZONE_API_CL | (1 << 10)) +#define gcdZONE_CL_PROGRAM (gcdZONE_API_CL | (1 << 11)) +#define gcdZONE_CL_SAMPLER (gcdZONE_API_CL | (1 << 12)) + +/* Subzones of VX API */ +#define gcdZONE_VX_ARRAY (gcdZONE_API_VX | (1 << 0)) +#define gcdZONE_VX_BINARY (gcdZONE_API_VX | (1 << 1)) +#define gcdZONE_VX_CONTEXT (gcdZONE_API_VX | (1 << 2)) +#define gcdZONE_VX_CONV (gcdZONE_API_VX | (1 << 3)) +#define gcdZONE_VX_DELAY (gcdZONE_API_VX | (1 << 4)) +#define gcdZONE_VX_DIST (gcdZONE_API_VX | (1 << 5)) +#define gcdZONE_VX_GPULAYER (gcdZONE_API_VX | (1 << 6)) +#define gcdZONE_VX_GRAPH (gcdZONE_API_VX | (1 << 7)) +#define gcdZONE_VX_IMAGE (gcdZONE_API_VX | (1 << 8)) +#define gcdZONE_VX_INTERFACE (gcdZONE_API_VX | (1 << 9)) +#define gcdZONE_VX_KERNEL (gcdZONE_API_VX | (1 << 10)) +#define gcdZONE_VX_LAYER (gcdZONE_API_VX | (1 << 11)) +#define gcdZONE_VX_LUT (gcdZONE_API_VX | (1 << 12)) +#define gcdZONE_VX_MATRIX (gcdZONE_API_VX | (1 << 13)) +#define gcdZONE_VX_MEMORY (gcdZONE_API_VX | (1 << 14)) +#define gcdZONE_VX_METAFMT (gcdZONE_API_VX | (1 << 15)) +#define gcdZONE_VX_NODE (gcdZONE_API_VX | (1 << 16)) +#define gcdZONE_VX_OBJARRAY (gcdZONE_API_VX | (1 << 17)) +#define gcdZONE_VX_PARAM (gcdZONE_API_VX | (1 << 18)) +#define gcdZONE_VX_PROGRAM (gcdZONE_API_VX | (1 << 19)) +#define gcdZONE_VX_PYRAMID (gcdZONE_API_VX | (1 << 20)) +#define gcdZONE_VX_REF (gcdZONE_API_VX | (1 << 21)) +#define gcdZONE_VX_REMAP (gcdZONE_API_VX | (1 << 22)) +#define gcdZONE_VX_SCALAR (gcdZONE_API_VX | (1 << 23)) +#define gcdZONE_VX_TARGET (gcdZONE_API_VX | (1 << 24)) +#define gcdZONE_VX_TENSOR (gcdZONE_API_VX | (1 << 25)) +#define gcdZONE_VX_THRESHOLD (gcdZONE_API_VX | (1 << 26)) +#define gcdZONE_VX_OTHERS (gcdZONE_API_VX | (1 << 27)) + +/******************************************************************************\ +******************************** Utility Zones ********************************* +\******************************************************************************/ + +/* Value for Disabling All Subzones */ +#define gcdZONE_NONE 0xF0000000 + +/* Value for Enabling All Subzones */ +#define gcdZONE_ALL 0x0FFFFFFF + + +/******************************************************************************\ +*********************************** END **************************************** +\******************************************************************************/ + +#ifdef __cplusplus +} +#endif + +#endif /* __gc_hal_debug_zones_h_ */ + +
diff --git a/hal/kernel/inc/gc_hal_drm.h b/hal/kernel/inc/gc_hal_drm.h new file mode 100644 index 0000000..a0635ee --- /dev/null +++ b/hal/kernel/inc/gc_hal_drm.h
@@ -0,0 +1,201 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __VIVNATE_DRM_H__ +#define __VIVNATE_DRM_H__ + + +#if defined(__cplusplus) +extern "C" { +#endif + +/* creation flag bits. */ +#define DRM_VIV_GEM_CONTIGUOUS (1u << 0) +#define DRM_VIV_GEM_CACHED (1u << 1) +#define DRM_VIV_GEM_SECURE (1u << 2) +#define DRM_VIV_GEM_CMA_LIMIT (1u << 3) + +struct drm_viv_gem_create { + __u64 size; + __u32 flags; + __u32 handle; +}; + +struct drm_viv_gem_lock { + __u32 handle; + __u32 cacheable; + __u64 logical; +}; + +struct drm_viv_gem_unlock { + __u32 handle; +}; + + +#define DRM_VIV_GEM_CLEAN_CACHE 0x01 +#define DRM_VIV_GEM_INVALIDATE_CACHE 0x02 +#define DRM_VIV_GEM_FLUSH_CACHE 0x03 +#define DRM_VIV_GEM_MEMORY_BARRIER 0x04 + +struct drm_viv_gem_cache { + __u32 handle; + __u32 op; + __u64 logical; + __u64 bytes; +}; + + +#define DRM_VIV_GEM_PARAM_POOL 0x00 +#define DRM_VIV_GEM_PARAM_SIZE 0x01 + +struct drm_viv_gem_query { + __u32 handle; + __u32 param; + __u64 value; +}; + + +struct drm_viv_gem_timestamp { + __u32 handle; + /* inc count, 0 for query current. */ + __u32 inc; + /* output inc'ed timestamp. */ + __u64 timestamp; +}; + + +/* basic tiling mode. */ +#define DRM_VIV_GEM_TILING_LINEAR 0x01 +#define DRM_VIV_GEM_TILING_TILED 0x02 +#define DRM_VIV_GEM_TILING_SUPERTILED 0x04 +#define DRM_VIV_GEM_TILING_MINORTILED 0x08 + +/* tiling mode modifiers. */ +#define DRM_VIV_GEM_TILING_SPLIT 0x10 +#define DRM_VIV_GEM_TILING_X_MAJOR 0x20 +#define DRM_VIV_GEM_TILING_Y_MAJOR 0x40 +#define DRM_VIV_GEM_TILING_SWAP 0x80 + +/* ts mode. */ +#define DRM_VIV_GEM_TS_NONE 0x00 +#define DRM_VIV_GEM_TS_DISABLED 0x01 +#define DRM_VIV_GEM_TS_NORMAL 0x02 +#define DRM_VIV_GEM_TS_COMPRESSED 0x03 + +struct drm_viv_gem_set_tiling { + __u32 handle; + __u32 tiling_mode; + + __u32 ts_mode; + __u64 clear_value; +}; + +struct drm_viv_gem_get_tiling { + __u32 handle; + __u32 tiling_mode; + + __u32 ts_mode; + __u64 clear_value; +}; + + +struct drm_viv_gem_attach_aux { + __u32 handle; + __u32 ts_handle; +}; + + +struct drm_viv_gem_ref_node { + __u32 handle; + + /* output. */ + __u32 node; + __u32 ts_node; +}; + + +#define DRM_VIV_GEM_CREATE 0x00 +#define DRM_VIV_GEM_LOCK 0x01 +#define DRM_VIV_GEM_UNLOCK 0x02 +#define DRM_VIV_GEM_CACHE 0x03 +#define DRM_VIV_GEM_QUERY 0x04 +#define DRM_VIV_GEM_TIMESTAMP 0x05 +#define DRM_VIV_GEM_SET_TILING 0x06 +#define DRM_VIV_GEM_GET_TILING 0x07 +#define DRM_VIV_GEM_ATTACH_AUX 0x08 +#define DRM_VIV_GEM_REF_NODE 0x09 +#define DRM_VIV_NUM_IOCTLS 0x0A + +#define DRM_IOCTL_VIV_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_CREATE, struct drm_viv_gem_create) +#define DRM_IOCTL_VIV_GEM_LOCK DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_LOCK, struct drm_viv_gem_lock) +#define DRM_IOCTL_VIV_GEM_UNLOCK DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_UNLOCK, struct drm_viv_gem_unlock) +#define DRM_IOCTL_VIV_GEM_CACHE DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_CACHE, struct drm_viv_gem_cache) +#define DRM_IOCTL_VIV_GEM_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_QUERY, struct drm_viv_gem_query) +#define DRM_IOCTL_VIV_GEM_TIMESTAMP DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_TIMESTAMP, struct drm_viv_gem_timestamp) +#define DRM_IOCTL_VIV_GEM_SET_TILING DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_SET_TILING, struct drm_viv_gem_set_tiling) +#define DRM_IOCTL_VIV_GEM_GET_TILING DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_GET_TILING, struct drm_viv_gem_get_tiling) +#define DRM_IOCTL_VIV_GEM_ATTACH_AUX DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_ATTACH_AUX, struct drm_viv_gem_attach_aux) +#define DRM_IOCTL_VIV_GEM_REF_NODE DRM_IOWR(DRM_COMMAND_BASE + DRM_VIV_GEM_REF_NODE, struct drm_viv_gem_ref_node) + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,4,0) +#define drm_gem_object_unreference_unlocked drm_gem_object_put_unlocked +#define drm_dev_unref drm_dev_put +#endif + +#if defined(__cplusplus) +} +#endif + +#endif /* __VIVNATE_DRM_H__ */
diff --git a/hal/kernel/inc/gc_hal_dump.h b/hal/kernel/inc/gc_hal_dump.h new file mode 100644 index 0000000..f64c0fe --- /dev/null +++ b/hal/kernel/inc/gc_hal_dump.h
@@ -0,0 +1,146 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_dump_h_ +#define __gc_hal_dump_h_ + +/* + gcdDUMP_KEY + + Set this to a string that appears in 'cat /proc/<pid>/cmdline'. E.g. 'camera'. + HAL will create dumps for the processes matching this key. +*/ +#ifndef gcdDUMP_KEY +# define gcdDUMP_KEY "process" +#endif + +/* + gcdDUMP_PATH + + The dump file location. Some processes cannot write to the sdcard. + Try apps' data dir, e.g. /data/data/com.android.launcher +*/ +#ifndef gcdDUMP_PATH +#if defined(ANDROID) +# define gcdDUMP_PATH "/mnt/sdcard/" +#else +# define gcdDUMP_PATH "./" +#endif +#endif + +/* + gcdDUMP_FILE_IN_KERNEL + + Default dump file for gcdDUMP_IN_KERNEL. + The file will be writen globally in kernel side. + Can be overwriten in runtime by debugfs:/gc/dump/dump_file + + 2 pseudo files: + [dmesg]: means dump to kernel debug message. + [ignored]: means dump ignored, nothing will be dumpped. + */ +#ifndef gcdDUMP_FILE_IN_KERNEL +# define gcdDUMP_FILE_IN_KERNEL "[dmesg]" +#endif + +/* + gcdDUMP_VERIFY_PER_DRAW + + Sub feature of gcdDUMP. + When set to 1, verify RT and images(if used) for every single draw to ease simulation debug. + Only valid for ES3 driver for now. +*/ +#ifndef gcdDUMP_VERIFY_PER_DRAW +# define gcdDUMP_VERIFY_PER_DRAW 0 +#endif + +/* Standalone dump features below. */ + +/* + gcdDUMP_FRAMERATE + When set to a value other than zero, averaqe frame rate will be dumped. + The value set is the starting frame that the average will be calculated. + This is needed because sometimes first few frames are too slow to be included + in the average. Frame count starts from 1. +*/ +#ifndef gcdDUMP_FRAMERATE +# define gcdDUMP_FRAMERATE 0 +#endif + + +/* + gcdDUMP_FRAME_TGA + + When set to a value other than 0, a dump of the frame specified by the value, + will be done into frame.tga. Frame count starts from 1. + */ +#ifndef gcdDUMP_FRAME_TGA +# define gcdDUMP_FRAME_TGA 0 +#endif + +/* + gcdDUMP_AHB_ACCESS + + When set to 1, a dump of all AHB register access will be printed to kernel + message. +*/ +#ifndef gcdDUMP_AHB_ACCESS +# define gcdDUMP_AHB_ACCESS 0 +#endif + +#endif /* __gc_hal_dump_h_ */ + + +
diff --git a/hal/kernel/inc/gc_hal_eglplatform.h b/hal/kernel/inc/gc_hal_eglplatform.h new file mode 100644 index 0000000..7655871 --- /dev/null +++ b/hal/kernel/inc/gc_hal_eglplatform.h
@@ -0,0 +1,589 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_eglplatform_h_ +#define __gc_hal_eglplatform_h_ + +#include "shared/gc_hal_types.h" +#include "gc_hal_base.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +#if defined(_WIN32) || defined(__VC32__) && !defined(__CYGWIN__) && !defined(__SCITECH_SNAP__) +#ifndef WIN32_LEAN_AND_MEAN +/* #define WIN32_LEAN_AND_MEAN 1 */ +#endif +#include <windows.h> + +typedef HDC HALNativeDisplayType; +typedef HWND HALNativeWindowType; +typedef HBITMAP HALNativePixmapType; + +typedef struct __BITFIELDINFO +{ + BITMAPINFO bmi; + RGBQUAD bmiColors[2]; +} +BITFIELDINFO; + +#elif /* defined(__APPLE__) || */ defined(__WINSCW__) || defined(__SYMBIAN32__) /* Symbian */ + +#elif defined(WL_EGL_PLATFORM) || defined(EGL_API_WL) /* Wayland */ + +#elif defined(__GBM__) /* GBM */ + +#elif defined(__ANDROID__) || defined(ANDROID) + +#elif defined(MIR_EGL_PLATFORM) /* Mir */ + +#elif defined(__QNXNTO__) + +#elif defined(__unix__) || defined(__APPLE__) + +#if defined(EGL_API_DFB) + +#elif defined(EGL_API_FB) + +#elif defined(EGL_API_NULLWS) + + +#else + +/* X11 (tetative). */ +#endif + +#else +#error "Platform not recognized" +#endif + +#if defined(_WIN32) || defined(__VC32__) && !defined(__CYGWIN__) && !defined(__SCITECH_SNAP__) + +#include "gc_hal_eglplatform_type.h" + +/******************************************************************************* +** Display. ******************************************************************** +*/ + +gceSTATUS +gcoOS_GetDisplay( + OUT HALNativeDisplayType * Display, + IN gctPOINTER Context + ); + +gceSTATUS +gcoOS_GetDisplayByIndex( + IN gctINT DisplayIndex, + OUT HALNativeDisplayType * Display, + IN gctPOINTER Context + ); + +gceSTATUS +gcoOS_GetDisplayInfo( + IN HALNativeDisplayType Display, + OUT gctINT * Width, + OUT gctINT * Height, + OUT gctSIZE_T * Physical, + OUT gctINT * Stride, + OUT gctINT * BitsPerPixel + ); + + + +gceSTATUS +gcoOS_GetDisplayInfoEx( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctUINT DisplayInfoSize, + OUT halDISPLAY_INFO * DisplayInfo + ); + +gceSTATUS +gcoOS_GetDisplayVirtual( + IN HALNativeDisplayType Display, + OUT gctINT * Width, + OUT gctINT * Height + ); + +gceSTATUS +gcoOS_GetDisplayBackbuffer( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + OUT gctPOINTER * context, + OUT gcoSURF * surface, + OUT gctUINT * Offset, + OUT gctINT * X, + OUT gctINT * Y + ); + +gceSTATUS +gcoOS_SetDisplayVirtual( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctUINT Offset, + IN gctINT X, + IN gctINT Y + ); + +gceSTATUS +gcoOS_SetDisplayVirtualEx( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctPOINTER Context, + IN gcoSURF Surface, + IN gctUINT Offset, + IN gctINT X, + IN gctINT Y + ); + +gceSTATUS +gcoOS_CancelDisplayBackbuffer( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctPOINTER Context, + IN gcoSURF Surface, + IN gctUINT Offset, + IN gctINT X, + IN gctINT Y + ); + +gceSTATUS +gcoOS_SetSwapInterval( + IN HALNativeDisplayType Display, + IN gctINT Interval +); + +gceSTATUS +gcoOS_SetSwapIntervalEx( + IN HALNativeDisplayType Display, + IN gctINT Interval, + IN gctPOINTER localDisplay); + +gceSTATUS +gcoOS_GetSwapInterval( + IN HALNativeDisplayType Display, + IN gctINT_PTR Min, + IN gctINT_PTR Max +); + +gceSTATUS +gcoOS_DisplayBufferRegions( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctINT NumRects, + IN gctINT_PTR Rects + ); + +gceSTATUS +gcoOS_DestroyDisplay( + IN HALNativeDisplayType Display + ); + +gceSTATUS +gcoOS_InitLocalDisplayInfo( + IN HALNativeDisplayType Display, + IN OUT gctPOINTER * localDisplay + ); + +gceSTATUS +gcoOS_DeinitLocalDisplayInfo( + IN HALNativeDisplayType Display, + IN OUT gctPOINTER * localDisplay + ); + +gceSTATUS +gcoOS_GetDisplayInfoEx2( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctPOINTER localDisplay, + IN gctUINT DisplayInfoSize, + OUT halDISPLAY_INFO * DisplayInfo + ); + +gceSTATUS +gcoOS_GetDisplayBackbufferEx( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctPOINTER localDisplay, + OUT gctPOINTER * context, + OUT gcoSURF * surface, + OUT gctUINT * Offset, + OUT gctINT * X, + OUT gctINT * Y + ); + +gceSTATUS +gcoOS_IsValidDisplay( + IN HALNativeDisplayType Display + ); + +gceSTATUS +gcoOS_GetNativeVisualId( + IN HALNativeDisplayType Display, + OUT gctINT* nativeVisualId + ); + +gctBOOL +gcoOS_SynchronousFlip( + IN HALNativeDisplayType Display + ); + +/******************************************************************************* +** Windows. ******************************************************************** +*/ + +gceSTATUS +gcoOS_CreateWindow( + IN HALNativeDisplayType Display, + IN gctINT X, + IN gctINT Y, + IN gctINT Width, + IN gctINT Height, + OUT HALNativeWindowType * Window + ); + +gceSTATUS +gcoOS_GetWindowInfo( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + OUT gctINT * X, + OUT gctINT * Y, + OUT gctINT * Width, + OUT gctINT * Height, + OUT gctINT * BitsPerPixel, + OUT gctUINT * Offset + ); + +gceSTATUS +gcoOS_DestroyWindow( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window + ); + +gceSTATUS +gcoOS_DrawImage( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctINT Left, + IN gctINT Top, + IN gctINT Right, + IN gctINT Bottom, + IN gctINT Width, + IN gctINT Height, + IN gctINT BitsPerPixel, + IN gctPOINTER Bits + ); + +gceSTATUS +gcoOS_GetImage( + IN HALNativeWindowType Window, + IN gctINT Left, + IN gctINT Top, + IN gctINT Right, + IN gctINT Bottom, + OUT gctINT * BitsPerPixel, + OUT gctPOINTER * Bits + ); + +gceSTATUS +gcoOS_GetWindowInfoEx( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + OUT gctINT * X, + OUT gctINT * Y, + OUT gctINT * Width, + OUT gctINT * Height, + OUT gctINT * BitsPerPixel, + OUT gctUINT * Offset, + OUT gceSURF_FORMAT * Format, + OUT gceSURF_TYPE * Type + ); + +gceSTATUS +gcoOS_DrawImageEx( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctINT Left, + IN gctINT Top, + IN gctINT Right, + IN gctINT Bottom, + IN gctINT Width, + IN gctINT Height, + IN gctINT BitsPerPixel, + IN gctPOINTER Bits, + IN gceSURF_FORMAT Format + ); + +/* + * Possiable types: + * gcvSURF_BITMAP + * gcvSURF_RENDER_TARGET + * gcvSURF_RENDER_TARGET_NO_COMPRESSION + * gcvSURF_RENDER_TARGET_NO_TILE_STATUS + */ +gceSTATUS +gcoOS_SetWindowFormat( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gceSURF_TYPE Type, + IN gceSURF_FORMAT Format + ); + + +/******************************************************************************* +** Pixmaps. ******************************************************************** +*/ + +gceSTATUS +gcoOS_CreatePixmap( + IN HALNativeDisplayType Display, + IN gctINT Width, + IN gctINT Height, + IN gctINT BitsPerPixel, + OUT HALNativePixmapType * Pixmap + ); + +gceSTATUS +gcoOS_GetPixmapInfo( + IN HALNativeDisplayType Display, + IN HALNativePixmapType Pixmap, + OUT gctINT * Width, + OUT gctINT * Height, + OUT gctINT * BitsPerPixel, + OUT gctINT * Stride, + OUT gctPOINTER * Bits + ); + +gceSTATUS +gcoOS_DrawPixmap( + IN HALNativeDisplayType Display, + IN HALNativePixmapType Pixmap, + IN gctINT Left, + IN gctINT Top, + IN gctINT Right, + IN gctINT Bottom, + IN gctINT Width, + IN gctINT Height, + IN gctINT BitsPerPixel, + IN gctPOINTER Bits + ); + +gceSTATUS +gcoOS_DestroyPixmap( + IN HALNativeDisplayType Display, + IN HALNativePixmapType Pixmap + ); + +gceSTATUS +gcoOS_GetPixmapInfoEx( + IN HALNativeDisplayType Display, + IN HALNativePixmapType Pixmap, + OUT gctINT * Width, + OUT gctINT * Height, + OUT gctINT * BitsPerPixel, + OUT gctINT * Stride, + OUT gctPOINTER * Bits, + OUT gceSURF_FORMAT * Format + ); + +gceSTATUS +gcoOS_CopyPixmapBits( + IN HALNativeDisplayType Display, + IN HALNativePixmapType Pixmap, + IN gctUINT DstWidth, + IN gctUINT DstHeight, + IN gctINT DstStride, + IN gceSURF_FORMAT DstFormat, + OUT gctPOINTER DstBits + ); + +/******************************************************************************* +** OS relative. **************************************************************** +*/ +gceSTATUS +gcoOS_LoadEGLLibrary( + OUT gctHANDLE * Handle + ); + +gceSTATUS +gcoOS_FreeEGLLibrary( + IN gctHANDLE Handle + ); + +gceSTATUS +gcoOS_ShowWindow( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window + ); + +gceSTATUS +gcoOS_HideWindow( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window + ); + +gceSTATUS +gcoOS_SetWindowTitle( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + IN gctCONST_STRING Title + ); + +gceSTATUS +gcoOS_CapturePointer( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window + ); + +gceSTATUS +gcoOS_GetEvent( + IN HALNativeDisplayType Display, + IN HALNativeWindowType Window, + OUT halEvent * Event + ); + +gceSTATUS +gcoOS_CreateClientBuffer( + IN gctINT Width, + IN gctINT Height, + IN gctINT Format, + IN gctINT Type, + OUT gctPOINTER * ClientBuffer + ); + +gceSTATUS +gcoOS_GetClientBufferInfo( + IN gctPOINTER ClientBuffer, + OUT gctINT * Width, + OUT gctINT * Height, + OUT gctINT * Stride, + OUT gctPOINTER * Bits + ); + +gceSTATUS +gcoOS_DestroyClientBuffer( + IN gctPOINTER ClientBuffer + ); + +gceSTATUS +gcoOS_DestroyContext( + IN gctPOINTER Display, + IN gctPOINTER Context + ); + +gceSTATUS +gcoOS_CreateContext( + IN gctPOINTER LocalDisplay, + IN gctPOINTER Context + ); + +gceSTATUS +gcoOS_MakeCurrent( + IN gctPOINTER LocalDisplay, + IN HALNativeWindowType DrawDrawable, + IN HALNativeWindowType ReadDrawable, + IN gctPOINTER Context, + IN gcoSURF ResolveTarget + ); + +gceSTATUS +gcoOS_CreateDrawable( + IN gctPOINTER LocalDisplay, + IN HALNativeWindowType Drawable + ); + +gceSTATUS +gcoOS_DestroyDrawable( + IN gctPOINTER LocalDisplay, + IN HALNativeWindowType Drawable + ); +gceSTATUS +gcoOS_SwapBuffers( + IN gctPOINTER LocalDisplay, + IN HALNativeWindowType Drawable, + IN gcoSURF RenderTarget, + IN gcoSURF ResolveTarget, + IN gctPOINTER ResolveBits, + OUT gctUINT *Width, + OUT gctUINT *Height + ); + +gceSTATUS +gcoOS_ResizeWindow( + IN gctPOINTER localDisplay, + IN HALNativeWindowType Drawable, + IN gctUINT Width, + IN gctUINT Height + ); + +gceSTATUS +gcoOS_RSForSwap( + IN gctPOINTER localDisplay, + IN HALNativeWindowType Drawable, + IN gctPOINTER resolve + ); + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __gc_hal_eglplatform_h_ */ + + +
diff --git a/hal/kernel/inc/gc_hal_eglplatform_type.h b/hal/kernel/inc/gc_hal_eglplatform_type.h new file mode 100644 index 0000000..4882df8 --- /dev/null +++ b/hal/kernel/inc/gc_hal_eglplatform_type.h
@@ -0,0 +1,184 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_eglplatform_type_h_ +#define __gc_hal_eglplatform_type_h_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Structure that defined keyboard mapping. */ +typedef struct _halKeyMap +{ + /* Normal key. */ + halKeys normal; + + /* Extended key. */ + halKeys extended; +} +halKeyMap; + +/* Event structure. */ +typedef struct _halEvent +{ + /* Event type. */ + halEventType type; + + /* Event data union. */ + union _halEventData + { + /* Event data for keyboard. */ + struct _halKeyboard + { + /* Scancode. */ + halKeys scancode; + + /* ASCII characte of the key pressed. */ + char key; + + /* Flag whether the key was pressed (1) or released (0). */ + char pressed; + } + keyboard; + + /* Event data for pointer. */ + struct _halPointer + { + /* Current pointer coordinate. */ + int x; + int y; + } + pointer; + + /* Event data for mouse buttons. */ + struct _halButton + { + /* Left button state. */ + int left; + + /* Middle button state. */ + int middle; + + /* Right button state. */ + int right; + + /* Current pointer coordinate. */ + int x; + int y; + } + button; + } + data; +} +halEvent; + +/* VFK_DISPLAY_INFO structure defining information returned by + vdkGetDisplayInfoEx. */ +typedef struct _halDISPLAY_INFO +{ + /* The size of the display in pixels. */ + int width; + int height; + + /* The stride of the dispay. -1 is returned if the stride is not known + ** for the specified display.*/ + int stride; + + /* The color depth of the display in bits per pixel. */ + int bitsPerPixel; + + /* The logical pointer to the display memory buffer. NULL is returned + ** if the pointer is not known for the specified display. */ + void * logical; + + /* The physical address of the display memory buffer. ~0 is returned + ** if the address is not known for the specified display. */ + unsigned long physical; + + /* Can be wraped as surface. */ + int wrapFB; + + /* FB_MULTI_BUFFER support */ + int multiBuffer; + int backBufferY; + + /* Tiled buffer / tile status support. */ + int tiledBuffer; + int tileStatus; + int compression; + + /* The color info of the display. */ + unsigned int alphaLength; + unsigned int alphaOffset; + unsigned int redLength; + unsigned int redOffset; + unsigned int greenLength; + unsigned int greenOffset; + unsigned int blueLength; + unsigned int blueOffset; + + /* Display flip support. */ + int flip; +} +halDISPLAY_INFO; + +#ifdef __cplusplus +} +#endif + +#endif /* __gc_hal_eglplatform_type_h_ */ + +
diff --git a/hal/kernel/inc/gc_hal_engine.h b/hal/kernel/inc/gc_hal_engine.h new file mode 100644 index 0000000..e859bd7 --- /dev/null +++ b/hal/kernel/inc/gc_hal_engine.h
@@ -0,0 +1,2780 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_engine_h_ +#define __gc_hal_engine_h_ + +#include "shared/gc_hal_types.h" +#include "gc_hal_enum.h" + + + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _gcsSURF_RESOLVE_ARGS +{ + gceHAL_ARG_VERSION version; + + union _gcsSURF_RESOLVE_ARGS_UNION + { + + struct _gcsSURF_RESOLVE_ARG_v2 + { + gctBOOL yInverted; + gctBOOL directCopy; + gctBOOL resample; + gctBOOL bUploadTex; /* used for upload tex.*/ + gctBOOL visualizeDepth; /* convert depth to visible color */ + gcsPOINT srcOrigin; + gcsPOINT dstOrigin; + gcsPOINT rectSize; + gctUINT numSlices; + gceENGINE engine; /* 3DBlit engine */ + gctBOOL gpuOnly; /* need only try HW path.*/ + + gctBOOL dump; /* need dump for verify */ + gctBOOL srcSwizzle; /* src surface format swizzle infomation */ + gctBOOL dstSwizzle; /* dst surface format swizzle infomation */ + gctBOOL srcCompressed; /* src compressed format*/ + gctBOOL dstCompressed; /* dst compressed format*/ + } v2; + } uArgs; +} +gcsSURF_RESOLVE_ARGS; + +typedef struct _gscBUFFER_VIEW +{ + gctUINT32 cmd; +}gcsBUFFER_VIEW, *gcsBUFFER_VIEW_PTR; + +typedef struct _gcsIMAGE_VIEW +{ + gctUINT32 cmd; +}gcsIMAGE_VIEW, *gcsIMAGE_VIEW_PTR; + +#if gcdENABLE_3D +/******************************************************************************\ +****************************** Object Declarations ***************************** +\******************************************************************************/ + +typedef struct _gcoSTREAM * gcoSTREAM; +typedef struct _gcoVERTEX * gcoVERTEX; +typedef struct _gcoTEXTURE * gcoTEXTURE; +typedef struct _gcoINDEX * gcoINDEX; +typedef struct _gcsVERTEX_ATTRIBUTES * gcsVERTEX_ATTRIBUTES_PTR; +typedef struct _gcoVERTEXARRAY * gcoVERTEXARRAY; +typedef struct _gcoBUFOBJ * gcoBUFOBJ; + +#define gcdATTRIBUTE_COUNT 32 +#define gcdVERTEXARRAY_POOL_CAPACITY 32 + +#define gcvPORGRAM_STAGE_GPIPE (gcvPROGRAM_STAGE_VERTEX_BIT | \ + gcvPROGRAM_STAGE_TCS_BIT | \ + gcvPROGRAM_STAGE_TES_BIT | \ + gcvPROGRAM_STAGE_GEOMETRY_BIT) + +/******************************************************************************\ +********************************* gcoHAL Object ********************************* +\******************************************************************************/ + +gceSTATUS +gcoHAL_QueryShaderCaps( + IN gcoHAL Hal, + OUT gctUINT * UnifiedUniforms, + OUT gctUINT * VertUniforms, + OUT gctUINT * FragUniforms, + OUT gctUINT * Varyings, + OUT gctUINT * ShaderCoreCount, + OUT gctUINT * ThreadCount, + OUT gctUINT * VertInstructionCount, + OUT gctUINT * FragInstructionCount + ); + +gceSTATUS +gcoHAL_QuerySamplerBase( + IN gcoHAL Hal, + OUT gctUINT32 * VertexCount, + OUT gctINT_PTR VertexBase, + OUT gctUINT32 * FragmentCount, + OUT gctINT_PTR FragmentBase + ); + +gceSTATUS +gcoHAL_QueryUniformBase( + IN gcoHAL Hal, + OUT gctUINT32 * VertexBase, + OUT gctUINT32 * FragmentBase + ); + +gceSTATUS +gcoHAL_QueryTextureCaps( + IN gcoHAL Hal, + OUT gctUINT * MaxWidth, + OUT gctUINT * MaxHeight, + OUT gctUINT * MaxDepth, + OUT gctBOOL * Cubic, + OUT gctBOOL * NonPowerOfTwo, + OUT gctUINT * VertexSamplers, + OUT gctUINT * PixelSamplers + ); + +gceSTATUS +gcoHAL_QueryTextureMaxAniso( + IN gcoHAL Hal, + OUT gctUINT * MaxAnisoValue + ); + +gceSTATUS +gcoHAL_QueryStreamCaps( + IN gcoHAL Hal, + OUT gctUINT32 * MaxAttributes, + OUT gctUINT32 * MaxStreamStride, + OUT gctUINT32 * NumberOfStreams, + OUT gctUINT32 * Alignment, + OUT gctUINT32 * MaxAttribOffset + ); + +/******************************************************************************\ +********************************* gcoSURF Object ******************************** +\******************************************************************************/ + +/*----------------------------------------------------------------------------*/ +/*--------------------------------- gcoSURF 3D --------------------------------*/ +typedef struct _gcsSURF_BLIT_ARGS +{ + gcoSURF srcSurface; + gctINT srcX, srcY, srcZ; + gctINT srcWidth, srcHeight, srcDepth; + gcoSURF dstSurface; + gctINT dstX, dstY, dstZ; + gctINT dstWidth, dstHeight, dstDepth; + gctBOOL xReverse; + gctBOOL yReverse; + gctBOOL scissorTest; + gcsRECT scissor; + gctUINT flags; + gctUINT srcNumSlice, dstNumSlice; + gctBOOL needDecode; +} +gcsSURF_BLIT_ARGS; + +typedef struct _gcsSURF_CLEAR_ARGS +{ + /* + ** Color to fill the color portion of the framebuffer when clear + ** is called. + */ + struct { + gcuVALUE r; + gcuVALUE g; + gcuVALUE b; + gcuVALUE a; + /* Color has multiple value type so we must specify it. */ + gceVALUE_TYPE valueType; + } color; + + gcuVALUE depth; + gctUINT stencil; + + gctUINT8 stencilMask; /* stencil bit-wise mask */ + gctBOOL depthMask; /* Depth Write Mask */ + gctUINT8 colorMask; /* 4-bit channel Mask: ABGR:MSB->LSB */ + gcsRECT_PTR clearRect; /* NULL means full clear */ + gceCLEAR flags; /* clear flags */ + + gctUINT32 offset; /* Offset in surface to cube/array/3D, obsolete in v2 version */ + +} gcsSURF_CLEAR_ARGS, *gcsSURF_CLEAR_ARGS_PTR; + + +typedef struct _gscSURF_BLITDRAW_BLIT +{ + gcoSURF srcSurface; + gcoSURF dstSurface; + gcsRECT srcRect; + gcsRECT dstRect; + gceTEXTURE_FILTER filterMode; + gctBOOL xReverse; + gctBOOL yReverse; + gctBOOL scissorEnabled; + gcsRECT scissor; +}gscSURF_BLITDRAW_BLIT; + +typedef gceSTATUS (* gctSPLIT_DRAW_FUNC_PTR)( + IN gctPOINTER gc, + IN gctPOINTER instantDraw, + IN gctPOINTER splitDrawInfo + ); + +typedef struct _gcsSPLIT_DRAW_INFO +{ + gceSPLIT_DRAW_TYPE splitDrawType; + gctSPLIT_DRAW_FUNC_PTR splitDrawFunc; + + union _gcsSPLIT_DRAW_UNION + { + /* This path will split many draw.*/ + struct __gcsSPLIT_DRAW_INFO_TCS + { + gctPOINTER indexPtr; + gctUINT indexPerPatch; + }info_tcs; + + /* This path split into two draw at most. + ** es11 path follow the old code, es30 path + ** add more info parameter to record + */ + struct __gcsSPLIT_DRAW_INFO_INDEX_FETCH + { + gctSIZE_T instanceCount; + gctSIZE_T splitCount; + gcePRIMITIVE splitPrimMode; + gctSIZE_T splitPrimCount; + }info_index_fetch; + }u; +} gcsSPLIT_DRAW_INFO, +*gcsSPLIT_DRAW_INFO_PTR; + +typedef struct _gscSURF_BLITDRAW_ARGS +{ + /* always the fist member */ + gceHAL_ARG_VERSION version; + + union _gcsSURF_BLITDRAW_ARGS_UNION + { + struct _gscSURF_BLITDRAW_ARG_v1 + { + /* Whether it's clear or blit operation, can be extended. */ + gceBLITDRAW_TYPE type; + + union _gscSURF_BLITDRAW_UNION + { + gscSURF_BLITDRAW_BLIT blit; + + struct _gscSURF_BLITDRAW_CLEAR + { + gcsSURF_CLEAR_ARGS clearArgs; + gcoSURF rtSurface; + gcoSURF dsSurface; + } clear; + } u; + } v1; + } uArgs; +} +gcsSURF_BLITDRAW_ARGS; + +typedef struct _gcsSURF_BLITBLT_ARGS +{ + gctCONST_POINTER buf; + gceSURF_FORMAT format; + gctUINT32 stride; + gcoSURF dstSurf; + gcsPOINT dstOrigin; + gcsPOINT rectSize; + gctUINT32 dstOffset; +} +gcsSURF_BLITBLT_ARGS; + + +/* CPU Blit with format (including linear <-> tile) conversion*/ +gceSTATUS +gcoSURF_BlitCPU( + gcsSURF_BLIT_ARGS* args + ); + +/* Copy a rectangular area with format conversion. */ +gceSTATUS +gcoSURF_CopyPixels( + IN gcsSURF_VIEW *SrcView, + IN gcsSURF_VIEW *DstView, + IN gcsSURF_RESOLVE_ARGS *Args + ); + +/* Clear surface function. */ +gceSTATUS +gcoSURF_Clear( + IN gcsSURF_VIEW *SurfView, + IN gcsSURF_CLEAR_ARGS_PTR ClearArgs + ); + +/* Preserve pixels from source. */ +gceSTATUS +gcoSURF_Preserve( + IN gcoSURF SrcSurf, + IN gcoSURF DstSurf, + IN gcsRECT_PTR MaskRect + ); + +/* TO BE REMOVED */ +gceSTATUS +depr_gcoSURF_Resolve( + IN gcoSURF SrcSurface, + IN gcoSURF DestSurface, + IN gctUINT32 DestAddress, + IN gctPOINTER DestBits, + IN gctINT DestStride, + IN gceSURF_TYPE DestType, + IN gceSURF_FORMAT DestFormat, + IN gctUINT DestWidth, + IN gctUINT DestHeight + ); + +gceSTATUS +depr_gcoSURF_ResolveRect( + IN gcoSURF SrcSurface, + IN gcoSURF DstSurface, + IN gctUINT32 DstAddress, + IN gctPOINTER DstBits, + IN gctINT DstStride, + IN gceSURF_TYPE DstType, + IN gceSURF_FORMAT DstFormat, + IN gctUINT DstWidth, + IN gctUINT DstHeight, + IN gcsPOINT_PTR SrcOrigin, + IN gcsPOINT_PTR gcoSURF, + IN gcsPOINT_PTR RectSize + ); + +/* Resample surface. */ +gceSTATUS +gcoSURF_Resample( + IN gcoSURF SrcSurf, + IN gcoSURF DstSurf, + IN gctBOOL sRGBDecode + ); + +/* Resolve rectangular area of a surface. */ +gceSTATUS +gcoSURF_ResolveRect( + IN gcsSURF_VIEW *SrcView, + IN gcsSURF_VIEW *DstView, + IN gcsSURF_RESOLVE_ARGS *Args + ); + +gceSTATUS +gcoSURF_GetResolveAlignment( + IN gcoSURF Surface, + OUT gctUINT *originX, + OUT gctUINT *originY, + OUT gctUINT *sizeX, + OUT gctUINT *sizeY + ); + +gceSTATUS +gcoSURF_IsHWResolveable( + IN gcoSURF SrcSurf, + IN gcoSURF DstSurf, + IN gcsPOINT_PTR SrcOrigin, + IN gcsPOINT_PTR DstOrigin, + IN gcsPOINT_PTR RectSize + ); + +/* Set surface resolvability. */ +gceSTATUS +gcoSURF_SetResolvability( + IN gcoSURF Surface, + IN gctBOOL Resolvable + ); + +gceSTATUS +gcoSURF_IsRenderable( + IN gcoSURF Surface + ); + +gceSTATUS +gcoSURF_IsFormatRenderableAsRT( + IN gcoSURF Surface + ); + +gceSTATUS +gcoBUFOBJ_GetFence( + IN gcoBUFOBJ BufObj, + IN gceFENCE_TYPE Type + ); + +gceSTATUS +gcoBUFOBJ_WaitFence( + IN gcoBUFOBJ BufObj, + IN gceFENCE_TYPE Type + ); + +gceSTATUS +gcoBUFOBJ_IsFenceEnabled( + IN gcoBUFOBJ BufObj + ); + +gceSTATUS +gcoSURF_GetFence( + IN gcoSURF Surface, + IN gceFENCE_TYPE Type + ); + +gceSTATUS +gcoSURF_WaitFence( + IN gcoSURF Surface + ); + +gceSTATUS +gcoSTREAM_GetFence( + IN gcoSTREAM stream + ); + +gceSTATUS +gcoSTREAM_WaitFence( + IN gcoSTREAM stream + ); + +gceSTATUS +gcoINDEX_GetFence( + IN gcoINDEX Index + ); + +gceSTATUS +gcoINDEX_WaitFence( + IN gcoINDEX Index, + IN gceFENCE_TYPE Type + ); + +gceSTATUS +gcoSURF_DrawBlit( + gcsSURF_VIEW *SrcView, + gcsSURF_VIEW *DstView, + gscSURF_BLITDRAW_BLIT *Args + ); + + +/******************************************************************************\ +******************************** gcoINDEX Object ******************************* +\******************************************************************************/ + +/* Construct a new gcoINDEX object. */ +gceSTATUS +gcoINDEX_Construct( + IN gcoHAL Hal, + OUT gcoINDEX * Index + ); + +/* Destroy a gcoINDEX object. */ +gceSTATUS +gcoINDEX_Destroy( + IN gcoINDEX Index + ); + +/* Lock index in memory. */ +gceSTATUS +gcoINDEX_Lock( + IN gcoINDEX Index, + OUT gctUINT32 * Address, + OUT gctPOINTER * Memory + ); + +/* Unlock index that was previously locked with gcoINDEX_Lock. */ +gceSTATUS +gcoINDEX_Unlock( + IN gcoINDEX Index + ); + +/* Upload index data into the memory. */ +gceSTATUS +gcoINDEX_Load( + IN gcoINDEX Index, + IN gceINDEX_TYPE IndexType, + IN gctUINT32 IndexCount, + IN gctPOINTER IndexBuffer + ); + +/* Bind an index object to the hardware. */ +gceSTATUS +gcoINDEX_Bind( + IN gcoINDEX Index, + IN gceINDEX_TYPE Type + ); + +/* Bind an index object to the hardware. */ +gceSTATUS +gcoINDEX_BindOffset( + IN gcoINDEX Index, + IN gceINDEX_TYPE Type, + IN gctUINT32 Offset + ); + +/* Free existing index buffer. */ +gceSTATUS +gcoINDEX_Free( + IN gcoINDEX Index + ); + +/* Upload data into an index buffer. */ +gceSTATUS +gcoINDEX_Upload( + IN gcoINDEX Index, + IN gctCONST_POINTER Buffer, + IN gctSIZE_T Bytes + ); + +/* Upload data into an index buffer starting at an offset. */ +gceSTATUS +gcoINDEX_UploadOffset( + IN gcoINDEX Index, + IN gctSIZE_T Offset, + IN gctCONST_POINTER Buffer, + IN gctSIZE_T Bytes + ); + +/*Merge index2 to index1 from 0, index2 must subset of inex1*/ +gceSTATUS +gcoINDEX_Merge( + IN gcoINDEX Index1, + IN gcoINDEX Index2 + ); + +/*check if index buffer is enough for this draw*/ +gctBOOL +gcoINDEX_CheckRange( + IN gcoINDEX Index, + IN gceINDEX_TYPE Type, + IN gctINT Count, + IN gctUINT32 Indices + ); + +/* Query the index capabilities. */ +gceSTATUS +gcoINDEX_QueryCaps( + OUT gctBOOL * Index8, + OUT gctBOOL * Index16, + OUT gctBOOL * Index32, + OUT gctUINT * MaxIndex + ); + +/* Determine the index range in the current index buffer. */ +gceSTATUS +gcoINDEX_GetIndexRange( + IN gcoINDEX Index, + IN gceINDEX_TYPE Type, + IN gctSIZE_T Offset, + IN gctUINT32 Count, + OUT gctUINT32 * MinimumIndex, + OUT gctUINT32 * MaximumIndex + ); + +/* Dynamic buffer management. */ +gceSTATUS +gcoINDEX_SetDynamic( + IN gcoINDEX Index, + IN gctSIZE_T Bytes, + IN gctUINT Buffers + ); + +gceSTATUS +gcoCLHardware_Construct(void); +/******************************************************************************\ +********************************** gco3D Object ********************************* +\******************************************************************************/ + +/* Construct a new gco3D object. */ +gceSTATUS +gco3D_Construct( + IN gcoHAL Hal, + IN gctBOOL Robust, + OUT gco3D * Engine + ); + +/* Destroy an gco3D object. */ +gceSTATUS +gco3D_Destroy( + IN gco3D Engine + ); + +/* Set 3D API type. */ +gceSTATUS +gco3D_SetAPI( + IN gco3D Engine, + IN gceAPI ApiType + ); + +/* Get 3D API type. */ +gceSTATUS +gco3D_GetAPI( + IN gco3D Engine, + OUT gceAPI * ApiType + ); + +gceSTATUS +gco3D_SetTarget( + IN gco3D Engine, + IN gctUINT32 TargetIndex, + IN gcsSURF_VIEW *SurfView, + IN gctUINT32 LayerIndex + ); + +gceSTATUS +gco3D_UnsetTarget( + IN gco3D Engine, + IN gctUINT32 TargetIndex, + IN gcoSURF Surface + ); + +gceSTATUS +gco3D_SetPSOutputMapping( + IN gco3D Engine, + IN gctINT32 * psOutputMapping + ); + +gceSTATUS +gco3D_SetRenderLayered( + IN gco3D Engine, + IN gctBOOL Enable, + IN gctUINT MaxLayers + ); + +gceSTATUS +gco3D_SetShaderLayered( + IN gco3D Engine, + IN gctBOOL Enable + ); + +gceSTATUS +gco3D_IsProgramSwitched( + IN gco3D Engine + ); + +/* Set depth buffer. */ +gceSTATUS +gco3D_SetDepth( + IN gco3D Engine, + IN gcsSURF_VIEW *SurfView + ); + +/* Unset depth buffer. */ +gceSTATUS +gco3D_UnsetDepth( + IN gco3D Engine, + IN gcoSURF Surface + ); + +/* Set viewport. */ +gceSTATUS +gco3D_SetViewport( + IN gco3D Engine, + IN gctINT32 Left, + IN gctINT32 Top, + IN gctINT32 Right, + IN gctINT32 Bottom + ); + +/* Set scissors. */ +gceSTATUS +gco3D_SetScissors( + IN gco3D Engine, + IN gctINT32 Left, + IN gctINT32 Top, + IN gctINT32 Right, + IN gctINT32 Bottom + ); + +/* Set clear color. */ +gceSTATUS +gco3D_SetClearColor( + IN gco3D Engine, + IN gctUINT8 Red, + IN gctUINT8 Green, + IN gctUINT8 Blue, + IN gctUINT8 Alpha + ); + +/* Set fixed point clear color. */ +gceSTATUS +gco3D_SetClearColorX( + IN gco3D Engine, + IN gctFIXED_POINT Red, + IN gctFIXED_POINT Green, + IN gctFIXED_POINT Blue, + IN gctFIXED_POINT Alpha + ); + +/* Set floating point clear color. */ +gceSTATUS +gco3D_SetClearColorF( + IN gco3D Engine, + IN gctFLOAT Red, + IN gctFLOAT Green, + IN gctFLOAT Blue, + IN gctFLOAT Alpha + ); + +/* Set fixed point clear depth. */ +gceSTATUS +gco3D_SetClearDepthX( + IN gco3D Engine, + IN gctFIXED_POINT Depth + ); + +/* Set floating point clear depth. */ +gceSTATUS +gco3D_SetClearDepthF( + IN gco3D Engine, + IN gctFLOAT Depth + ); + +/* Set clear stencil. */ +gceSTATUS +gco3D_SetClearStencil( + IN gco3D Engine, + IN gctUINT32 Stencil + ); + +/* Set shading mode. */ +gceSTATUS +gco3D_SetShading( + IN gco3D Engine, + IN gceSHADING Shading + ); + +/* Set blending mode. */ +gceSTATUS +gco3D_EnableBlending( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Set blending function. */ +gceSTATUS +gco3D_SetBlendFunction( + IN gco3D Engine, + IN gceBLEND_UNIT Unit, + IN gceBLEND_FUNCTION FunctionRGB, + IN gceBLEND_FUNCTION FunctionAlpha + ); + +/* Set blending mode. */ +gceSTATUS +gco3D_SetBlendMode( + IN gco3D Engine, + IN gceBLEND_MODE ModeRGB, + IN gceBLEND_MODE ModeAlpha + ); + +/* Set blending mode for separate rt control */ +gceSTATUS +gco3D_EnableBlendingIndexed( + IN gco3D Engine, + IN gctUINT Index, + IN gctBOOL Enable + ); + +/* Set blending function for separate rt control */ +gceSTATUS +gco3D_SetBlendFunctionIndexed( + IN gco3D Engine, + IN gctUINT Index, + IN gceBLEND_UNIT Unit, + IN gceBLEND_FUNCTION FunctionRGB, + IN gceBLEND_FUNCTION FunctionAlpha + ); + +/* Set blending mode for separate rt control*/ +gceSTATUS +gco3D_SetBlendModeIndexed( + IN gco3D Engine, + IN gctUINT Index, + IN gceBLEND_MODE ModeRGB, + IN gceBLEND_MODE ModeAlpha + ); + +/* Set blending color. */ +gceSTATUS +gco3D_SetBlendColor( + IN gco3D Engine, + IN gctUINT Red, + IN gctUINT Green, + IN gctUINT Blue, + IN gctUINT Alpha + ); + +/* Set fixed point blending color. */ +gceSTATUS +gco3D_SetBlendColorX( + IN gco3D Engine, + IN gctFIXED_POINT Red, + IN gctFIXED_POINT Green, + IN gctFIXED_POINT Blue, + IN gctFIXED_POINT Alpha + ); + +/* Set floating point blending color. */ +gceSTATUS +gco3D_SetBlendColorF( + IN gco3D Engine, + IN gctFLOAT Red, + IN gctFLOAT Green, + IN gctFLOAT Blue, + IN gctFLOAT Alpha + ); + +/* Set culling mode. */ +gceSTATUS +gco3D_SetCulling( + IN gco3D Engine, + IN gceCULL Mode + ); + +/* Enable point size */ +gceSTATUS +gco3D_SetPointSizeEnable( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Set point sprite */ +gceSTATUS +gco3D_SetPointSprite( + IN gco3D Engine, + IN gctBOOL Enable + ); + + +/* Enable/Disable primitive-id. */ +gceSTATUS +gco3D_SetPrimitiveIdEnable( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Set fill mode. */ +gceSTATUS +gco3D_SetFill( + IN gco3D Engine, + IN gceFILL Mode + ); + +/* Set depth compare mode. */ +gceSTATUS +gco3D_SetDepthCompare( + IN gco3D Engine, + IN gceCOMPARE Compare + ); + +/* Enable depth writing. */ +gceSTATUS +gco3D_EnableDepthWrite( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Set depth mode. */ +gceSTATUS +gco3D_SetDepthMode( + IN gco3D Engine, + IN gceDEPTH_MODE Mode + ); + +/* Set depth range. */ +gceSTATUS +gco3D_SetDepthRangeX( + IN gco3D Engine, + IN gceDEPTH_MODE Mode, + IN gctFIXED_POINT Near, + IN gctFIXED_POINT Far + ); + +/* Set depth range. */ +gceSTATUS +gco3D_SetDepthRangeF( + IN gco3D Engine, + IN gceDEPTH_MODE Mode, + IN gctFLOAT Near, + IN gctFLOAT Far + ); + +/* Set last pixel enable */ +gceSTATUS +gco3D_SetLastPixelEnable( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Set depth Bias and Scale */ +gceSTATUS +gco3D_SetDepthScaleBiasX( + IN gco3D Engine, + IN gctFIXED_POINT DepthScale, + IN gctFIXED_POINT DepthBias + ); + +gceSTATUS +gco3D_SetDepthScaleBiasF( + IN gco3D Engine, + IN gctFLOAT DepthScale, + IN gctFLOAT DepthBias + ); + +/* Set depth near and far clipping plane. */ +gceSTATUS +gco3D_SetDepthPlaneF( + IN gco3D Engine, + IN gctFLOAT Near, + IN gctFLOAT Far + ); + +/* Enable or disable dithering. */ +gceSTATUS +gco3D_EnableDither( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Set color write enable bits. */ +gceSTATUS +gco3D_SetColorWrite( + IN gco3D Engine, + IN gctUINT8 Enable + ); + +/* Set color write enable bits for separate rt control */ +gceSTATUS +gco3D_SetColorWriteIndexed( + IN gco3D Engine, + IN gctUINT Index, + IN gctUINT8 Enable + ); + +/* Enable or disable early depth. */ +gceSTATUS +gco3D_SetEarlyDepth( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Deprecated: Enable or disable all early depth operations. */ +gceSTATUS +gco3D_SetAllEarlyDepthModes( + IN gco3D Engine, + IN gctBOOL Disable + ); + + +gceSTATUS +gco3D_SetEarlyDepthFromAPP( + IN gco3D Engine, + IN gctBOOL EarlyDepthFromAPP + ); + +gceSTATUS +gco3D_SetRADepthWrite( + IN gco3D Engine, + IN gctBOOL Disable, + IN gctBOOL psReadZ, + IN gctBOOL psReadW + ); + +gceSTATUS +gco3D_SetPatchVertices( + IN gco3D Engine, + IN gctINT PatchVertices + ); + + +/* Switch dynamic early mode */ +gceSTATUS +gco3D_SwitchDynamicEarlyDepthMode( + IN gco3D Engine + ); + +/* Set dynamic early mode */ +gceSTATUS +gco3D_DisableDynamicEarlyDepthMode( + IN gco3D Engine, + IN gctBOOL Disable + ); + +/* Enable or disable depth-only mode. */ +gceSTATUS +gco3D_SetDepthOnly( + IN gco3D Engine, + IN gctBOOL Enable + ); + +typedef struct _gcsSTENCIL_INFO * gcsSTENCIL_INFO_PTR; +typedef struct _gcsSTENCIL_INFO +{ + gceSTENCIL_MODE mode; + + gctUINT8 maskFront; + gctUINT8 maskBack; + gctUINT8 writeMaskFront; + gctUINT8 writeMaskBack; + + gctUINT8 referenceFront; + + gceCOMPARE compareFront; + gceSTENCIL_OPERATION passFront; + gceSTENCIL_OPERATION failFront; + gceSTENCIL_OPERATION depthFailFront; + + gctUINT8 referenceBack; + gceCOMPARE compareBack; + gceSTENCIL_OPERATION passBack; + gceSTENCIL_OPERATION failBack; + gceSTENCIL_OPERATION depthFailBack; +} +gcsSTENCIL_INFO; + +/* Set stencil mode. */ +gceSTATUS +gco3D_SetStencilMode( + IN gco3D Engine, + IN gceSTENCIL_MODE Mode + ); + +/* Set stencil mask. */ +gceSTATUS +gco3D_SetStencilMask( + IN gco3D Engine, + IN gctUINT8 Mask + ); + +/* Set stencil back mask. */ +gceSTATUS +gco3D_SetStencilMaskBack( + IN gco3D Engine, + IN gctUINT8 Mask + ); + +/* Set stencil write mask. */ +gceSTATUS +gco3D_SetStencilWriteMask( + IN gco3D Engine, + IN gctUINT8 Mask + ); + +/* Set stencil back write mask. */ +gceSTATUS +gco3D_SetStencilWriteMaskBack( + IN gco3D Engine, + IN gctUINT8 Mask + ); + +/* Set stencil reference. */ +gceSTATUS +gco3D_SetStencilReference( + IN gco3D Engine, + IN gctUINT8 Reference, + IN gctBOOL Front + ); + +/* Set stencil compare. */ +gceSTATUS +gco3D_SetStencilCompare( + IN gco3D Engine, + IN gceSTENCIL_WHERE Where, + IN gceCOMPARE Compare + ); + +/* Set stencil operation on pass. */ +gceSTATUS +gco3D_SetStencilPass( + IN gco3D Engine, + IN gceSTENCIL_WHERE Where, + IN gceSTENCIL_OPERATION Operation + ); + +/* Set stencil operation on fail. */ +gceSTATUS +gco3D_SetStencilFail( + IN gco3D Engine, + IN gceSTENCIL_WHERE Where, + IN gceSTENCIL_OPERATION Operation + ); + +/* Set stencil operation on depth fail. */ +gceSTATUS +gco3D_SetStencilDepthFail( + IN gco3D Engine, + IN gceSTENCIL_WHERE Where, + IN gceSTENCIL_OPERATION Operation + ); + +/* Set all stencil states in one blow. */ +gceSTATUS +gco3D_SetStencilAll( + IN gco3D Engine, + IN gcsSTENCIL_INFO_PTR Info + ); + +typedef struct _gcsALPHA_INFO * gcsALPHA_INFO_PTR; +typedef struct _gcsALPHA_INFO +{ + /* Alpha test states. */ + gctBOOL test; + gceCOMPARE compare; + gctUINT8 reference; + gctFLOAT floatReference; + + /* Alpha blending states. */ + gctBOOL blend[gcdMAX_DRAW_BUFFERS]; + + gceBLEND_FUNCTION srcFuncColor[gcdMAX_DRAW_BUFFERS]; + gceBLEND_FUNCTION srcFuncAlpha[gcdMAX_DRAW_BUFFERS]; + gceBLEND_FUNCTION trgFuncColor[gcdMAX_DRAW_BUFFERS]; + gceBLEND_FUNCTION trgFuncAlpha[gcdMAX_DRAW_BUFFERS]; + + gceBLEND_MODE modeColor[gcdMAX_DRAW_BUFFERS]; + gceBLEND_MODE modeAlpha[gcdMAX_DRAW_BUFFERS]; + + gctUINT32 color; + + gctBOOL anyBlendEnabled; +} +gcsALPHA_INFO; + + +/* Enable or disable alpha test. */ +gceSTATUS +gco3D_SetAlphaTest( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Set alpha test compare. */ +gceSTATUS +gco3D_SetAlphaCompare( + IN gco3D Engine, + IN gceCOMPARE Compare + ); + +/* Set alpha test reference in unsigned integer. */ +gceSTATUS +gco3D_SetAlphaReference( + IN gco3D Engine, + IN gctUINT8 Reference, + IN gctFLOAT FloatReference + ); + +/* Set alpha test reference in fixed point. */ +gceSTATUS +gco3D_SetAlphaReferenceX( + IN gco3D Engine, + IN gctFIXED_POINT Reference + ); + +/* Set alpha test reference in floating point. */ +gceSTATUS +gco3D_SetAlphaReferenceF( + IN gco3D Engine, + IN gctFLOAT Reference + ); + +#if gcdALPHA_KILL_IN_SHADER +gceSTATUS +gco3D_SetAlphaKill( + IN gco3D Engine, + IN gctBOOL AlphaKill, + IN gctBOOL ColorKill + ); +#endif + +/* Enable/Disable anti-alias line. */ +gceSTATUS +gco3D_SetAntiAliasLine( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Set texture slot for anti-alias line. */ +gceSTATUS +gco3D_SetAALineTexSlot( + IN gco3D Engine, + IN gctUINT TexSlot + ); + +/* Set anti-alias line width scale. */ +gceSTATUS +gco3D_SetAALineWidth( + IN gco3D Engine, + IN gctFLOAT Width + ); + +/* Draw a number of primitives. */ +gceSTATUS +gco3D_DrawPrimitives( + IN gco3D Engine, + IN gcePRIMITIVE Type, + IN gctSIZE_T StartVertex, + IN gctSIZE_T PrimitiveCount + ); + +gceSTATUS +gco3D_DrawIndirectPrimitives( + IN gco3D Engine, + IN gcePRIMITIVE Type, + IN gctBOOL DrawIndex, + IN gctINT BaseOffset, + IN gcoBUFOBJ BufObj + ); + +gceSTATUS +gco3D_MultiDrawIndirectPrimitives( + IN gco3D Engine, + IN gcePRIMITIVE Type, + IN gctBOOL DrawIndex, + IN gctINT BaseOffset, + IN gctINT DrawCount, + IN gctINT Stride, + IN gcoBUFOBJ BufObj + ); + +gceSTATUS +gco3D_DrawInstancedPrimitives( + IN gco3D Engine, + IN gcePRIMITIVE Type, + IN gctBOOL DrawIndex, + IN gctINT StartVertex, + IN gctSIZE_T StartIndex, + IN gctSIZE_T PrimitiveCount, + IN gctSIZE_T VertexCount, + IN gctSIZE_T InstanceCount + ); + +gceSTATUS +gco3D_DrawNullPrimitives( + IN gco3D Engine + ); + +gceSTATUS +gco3D_DrawPrimitivesCount( + IN gco3D Engine, + IN gcePRIMITIVE Type, + IN gctINT* StartVertex, + IN gctSIZE_T* VertexCount, + IN gctSIZE_T PrimitiveCount + ); + + +/* Draw a number of primitives using offsets. */ +gceSTATUS +gco3D_DrawPrimitivesOffset( + IN gco3D Engine, + IN gcePRIMITIVE Type, + IN gctINT32 StartOffset, + IN gctSIZE_T PrimitiveCount + ); + +/* Draw a number of indexed primitives. */ +gceSTATUS +gco3D_DrawIndexedPrimitives( + IN gco3D Engine, + IN gcePRIMITIVE Type, + IN gctSIZE_T BaseVertex, + IN gctSIZE_T StartIndex, + IN gctSIZE_T PrimitiveCount + ); + +/* Draw a number of indexed primitives using offsets. */ +gceSTATUS +gco3D_DrawIndexedPrimitivesOffset( + IN gco3D Engine, + IN gcePRIMITIVE Type, + IN gctINT32 BaseOffset, + IN gctINT32 StartOffset, + IN gctSIZE_T PrimitiveCount + ); + +/* Draw a element from pattern */ +gceSTATUS +gco3D_DrawPattern( + IN gco3D Engine, + IN gcsFAST_FLUSH_PTR FastFlushInfo + ); + +/* Enable or disable anti-aliasing. */ +gceSTATUS +gco3D_SetAntiAlias( + IN gco3D Engine, + IN gctBOOL Enable + ); + +/* Set msaa samples */ +gceSTATUS +gco3D_SetSamples( + IN gco3D Engine, + IN gctUINT32 Samples + ); + + +/* Write data into the command buffer. */ +gceSTATUS +gco3D_WriteBuffer( + IN gco3D Engine, + IN gctCONST_POINTER Data, + IN gctSIZE_T Bytes, + IN gctBOOL Aligned + ); + +/* Send sempahore and stall until sempahore is signalled. */ +gceSTATUS +gco3D_Semaphore( + IN gco3D Engine, + IN gceWHERE From, + IN gceWHERE To, + IN gceHOW How); + + +/* Explicitly flush shader L1 cache */ +gceSTATUS +gco3D_FlushSHL1Cache( + IN gco3D Engine + ); + +/* Set the subpixels center. */ +gceSTATUS +gco3D_SetCentroids( + IN gco3D Engine, + IN gctUINT32 Index, + IN gctPOINTER Centroids + ); + +/* query msaa sample coordinates */ +gceSTATUS +gco3D_GetSampleCoords( + IN gco3D Engine, + IN gctUINT32 SampleIndex, + IN gctBOOL yInverted, + OUT gctFLOAT_PTR Coords + ); + +gceSTATUS +gco3D_SetLogicOp( + IN gco3D Engine, + IN gctUINT8 Rop + ); + +gceSTATUS +gco3D_SetQuery( + IN gco3D Engine, + IN gctUINT32 QueryHeader, + IN gceQueryType Type, + IN gctBOOL Enable, + IN gctUINT32 Index + ); + +gceSTATUS +gco3D_GetQuery( + IN gco3D Engine, + IN gceQueryType Type, + IN gcsSURF_NODE_PTR Node, + IN gctUINT32 Size, + IN gctPOINTER Locked, + IN gctUINT32 IndexedId, + OUT gctINT32 * Index + ); + +gceSTATUS +gco3D_SetXfbHeader( + IN gco3D Engine, + IN gctUINT32 Physical + ); + +gceSTATUS +gco3D_SetXfbBuffer( + IN gco3D Engine, + IN gctUINT32 Index, + IN gctUINT32 BufferAddr, + IN gctUINT32 BufferStride, + IN gctUINT32 BufferSize + ); + +gceSTATUS +gco3D_SetXfbCmd( + IN gco3D Engine, + IN gceXfbCmd Cmd + ); + +gceSTATUS +gco3D_SetRasterDiscard( + IN gco3D Engine, + IN gctBOOL Enable + ); + +gceSTATUS +gco3D_SetColorOutCount( + IN gco3D Engine, + IN gctUINT32 ColorOutCount + ); + +gceSTATUS +gco3D_SetColorCacheMode( + IN gco3D Engine + ); + +gceSTATUS +gco3D_Set3DEngine( + IN gco3D Engine + ); + +gceSTATUS +gco3D_UnSet3DEngine( + IN gco3D Engine + ); + +gceSTATUS +gco3D_Get3DEngine( + OUT gco3D * Engine + ); + +gceSTATUS +gco3D_QueryReset( + IN gco3D Engine, + OUT gctBOOL_PTR Innocent + ); + +/* OCL thread walker information. */ +typedef struct _gcsTHREAD_WALKER_INFO * gcsTHREAD_WALKER_INFO_PTR; +typedef struct _gcsTHREAD_WALKER_INFO +{ + gctUINT32 dimensions; + gctUINT32 traverseOrder; + gctUINT32 enableSwathX; + gctUINT32 enableSwathY; + gctUINT32 enableSwathZ; + gctUINT32 swathSizeX; + gctUINT32 swathSizeY; + gctUINT32 swathSizeZ; + gctUINT32 valueOrder; + + gctUINT32 globalSizeX; + gctUINT32 globalOffsetX; + gctUINT32 globalSizeY; + gctUINT32 globalOffsetY; + gctUINT32 globalSizeZ; + gctUINT32 globalOffsetZ; + + gctUINT32 globalScaleX; + gctUINT32 globalScaleY; + gctUINT32 globalScaleZ; + + gctUINT32 workGroupSizeX; + gctUINT32 workGroupCountX; + gctUINT32 workGroupSizeY; + gctUINT32 workGroupCountY; + gctUINT32 workGroupSizeZ; + gctUINT32 workGroupCountZ; + + gctUINT32 threadAllocation; + gctBOOL barrierUsed; + gctUINT32 memoryAccessFlag; /* same as gceMEMORY_ACCESS_FLAG */ + gctBOOL indirect; + gctUINT32 groupNumberUniformIdx; + gctUINT32 baseAddress; + gctBOOL bDual16; +} +gcsTHREAD_WALKER_INFO; + +#if gcdENABLE_3D && gcdUSE_VX +/* VX thread walker parameters. */ +typedef struct _gcsVX_THREAD_WALKER_PARAMETERS * gcsVX_THREAD_WALKER_PARAMETERS_PTR; + +typedef struct _gcsVX_THREAD_WALKER_PARAMETERS +{ + gctUINT32 valueOrder; + gctUINT32 workDim; + + gctUINT32 workGroupSizeX; + gctUINT32 workGroupCountX; + + gctUINT32 workGroupSizeY; + gctUINT32 workGroupCountY; + + gctUINT32 globalOffsetX; + gctUINT32 globalScaleX; + + gctUINT32 globalOffsetY; + gctUINT32 globalScaleY; + +#if gcdVX_OPTIMIZER > 1 + gctBOOL tileMode; +#endif +} +gcsVX_THREAD_WALKER_PARAMETERS; + +typedef struct _gcsVX_IMAGE_INFO * gcsVX_IMAGE_INFO_PTR; + +typedef struct _gcsVX_IMAGE_INFO +{ + gctUINT32 format; + gctUINT32 rect[4]; + gctUINT32 width; + gctUINT32 height; + + /*arraySize, sliceSize is for imageArray / image3D */ + gctUINT32 arraySize; + gctUINT32 sliceSize; + + gctUINT32 bpp; + gctUINT32 planes; + gctUINT32 componentCount; + gctBOOL isFloat; + + gctUINT32 uPixels; + gctUINT32 vPixels; + gceSURF_FORMAT internalFormat; + gctUINT32 border; + + /*vx_imagepatch_addressing_t == (gctUINT32 * 8) */ + gctUINT32 imagepatch[8 * 3]; + void *base_addr[3]; + + gctUINT32 stride[3]; + + gctPOINTER logicals[3]; + gctUINT32 physicals[3]; + gctUINT32 bytes; + + gcsSURF_NODE_PTR nodes[3]; + + gctBOOL isVXC; +#if gcdVX_OPTIMIZER + gctUINT32 uniformData[3][4]; +#endif +} +gcsVX_IMAGE_INFO; +typedef struct _gcsVX_DISTRIBUTION_INFO * gcsVX_DISTRIBUTION_INFO_PTR; + +typedef struct _gcsVX_DISTRIBUTION_INFO +{ + + gctUINT32 logical; + gctUINT32 physical; + gctUINT32 bytes; + + gcsSURF_NODE_PTR node; +} +gcsVX_DISTRIBUTION_INFO; +#endif + +/* Start OCL thread walker. */ +gceSTATUS +gco3D_InvokeThreadWalker( + IN gco3D Engine, + IN gcsTHREAD_WALKER_INFO_PTR Info + ); + +gceSTATUS +gco3D_GetClosestRenderFormat( + IN gco3D Engine, + IN gceSURF_FORMAT InFormat, + OUT gceSURF_FORMAT* OutFormat + ); + +/* Set w clip and w plane limit value. */ +gceSTATUS +gco3D_SetWClipEnable( + IN gco3D Engine, + IN gctBOOL Enable + ); + +gceSTATUS +gco3D_GetWClipEnable( + IN gco3D Engine, + OUT gctBOOL * Enable + ); + +gceSTATUS +gco3D_SetWPlaneLimitF( + IN gco3D Engine, + IN gctFLOAT Value + ); + +gceSTATUS +gco3D_SetWPlaneLimitX( + IN gco3D Engine, + IN gctFIXED_POINT Value + ); + +gceSTATUS +gco3D_SetWPlaneLimit( + IN gco3D Engine, + IN gctFLOAT Value + ); + +gceSTATUS +gco3D_PrimitiveRestart( + IN gco3D Engine, + IN gctBOOL PrimitiveRestart + ); + +gceSTATUS +gco3D_LoadProgram( + IN gco3D Engine, + IN gcePROGRAM_STAGE_BIT StageBits, + IN gctPOINTER ProgramState + ); + +gceSTATUS +gco3D_EnableAlphaToCoverage( + IN gco3D Engine, + IN gctBOOL Enable + ); + +gceSTATUS +gco3D_EnableSampleCoverage( + IN gco3D Engine, + IN gctBOOL Enable + ); + +gceSTATUS +gco3D_SetSampleCoverageValue( + IN gco3D Engine, + IN gctFLOAT CoverageValue, + IN gctBOOL Invert + ); + +gceSTATUS +gco3D_EnableSampleMask( + IN gco3D Engine, + IN gctBOOL Enable + ); + +gceSTATUS +gco3D_SetSampleMask( + IN gco3D Engine, + IN gctUINT32 SampleMask + ); + +gceSTATUS +gco3D_EnableSampleShading( + IN gco3D Engine, + IN gctBOOL Enable + ); + +gceSTATUS +gco3D_SetMinSampleShadingValue( + IN gco3D Engine, + IN gctFLOAT MinSampleShadingValue + ); + +gceSTATUS +gco3D_SetSampleShading( + IN gco3D Engine, + IN gctBOOL Enable, + IN gctBOOL IsSampleIn, + IN gctFLOAT SampleShadingValue + ); + +gceSTATUS +gco3D_EnableSampleMaskOut( + IN gco3D Engine, + IN gctBOOL Enable, + IN gctINT SampleMaskLoc + ); + +/*----------------------------------------------------------------------------*/ +/*-------------------------- gco3D Fragment Processor ------------------------*/ + +/* Set the fragment processor configuration. */ +gceSTATUS +gco3D_SetFragmentConfiguration( + IN gco3D Engine, + IN gctBOOL ColorFromStream, + IN gctBOOL EnableFog, + IN gctBOOL EnableSmoothPoint, + IN gctUINT32 ClipPlanes + ); + +/* Enable/disable texture stage operation. */ +gceSTATUS +gco3D_EnableTextureStage( + IN gco3D Engine, + IN gctINT Stage, + IN gctBOOL Enable + ); + +/* Program the channel enable masks for the color texture function. */ +gceSTATUS +gco3D_SetTextureColorMask( + IN gco3D Engine, + IN gctINT Stage, + IN gctBOOL ColorEnabled, + IN gctBOOL AlphaEnabled + ); + +/* Program the channel enable masks for the alpha texture function. */ +gceSTATUS +gco3D_SetTextureAlphaMask( + IN gco3D Engine, + IN gctINT Stage, + IN gctBOOL ColorEnabled, + IN gctBOOL AlphaEnabled + ); + +/* Program the constant fragment color. */ +gceSTATUS +gco3D_SetFragmentColorX( + IN gco3D Engine, + IN gctFIXED_POINT Red, + IN gctFIXED_POINT Green, + IN gctFIXED_POINT Blue, + IN gctFIXED_POINT Alpha + ); + +gceSTATUS +gco3D_SetFragmentColorF( + IN gco3D Engine, + IN gctFLOAT Red, + IN gctFLOAT Green, + IN gctFLOAT Blue, + IN gctFLOAT Alpha + ); + +/* Program the constant fog color. */ +gceSTATUS +gco3D_SetFogColorX( + IN gco3D Engine, + IN gctFIXED_POINT Red, + IN gctFIXED_POINT Green, + IN gctFIXED_POINT Blue, + IN gctFIXED_POINT Alpha + ); + +gceSTATUS +gco3D_SetFogColorF( + IN gco3D Engine, + IN gctFLOAT Red, + IN gctFLOAT Green, + IN gctFLOAT Blue, + IN gctFLOAT Alpha + ); + +/* Program the constant texture color. */ +gceSTATUS +gco3D_SetTetxureColorX( + IN gco3D Engine, + IN gctINT Stage, + IN gctFIXED_POINT Red, + IN gctFIXED_POINT Green, + IN gctFIXED_POINT Blue, + IN gctFIXED_POINT Alpha + ); + +gceSTATUS +gco3D_SetTetxureColorF( + IN gco3D Engine, + IN gctINT Stage, + IN gctFLOAT Red, + IN gctFLOAT Green, + IN gctFLOAT Blue, + IN gctFLOAT Alpha + ); + +/* Configure color texture function. */ +gceSTATUS +gco3D_SetColorTextureFunction( + IN gco3D Engine, + IN gctINT Stage, + IN gceTEXTURE_FUNCTION Function, + IN gceTEXTURE_SOURCE Source0, + IN gceTEXTURE_CHANNEL Channel0, + IN gceTEXTURE_SOURCE Source1, + IN gceTEXTURE_CHANNEL Channel1, + IN gceTEXTURE_SOURCE Source2, + IN gceTEXTURE_CHANNEL Channel2, + IN gctINT Scale + ); + +/* Configure alpha texture function. */ +gceSTATUS +gco3D_SetAlphaTextureFunction( + IN gco3D Engine, + IN gctINT Stage, + IN gceTEXTURE_FUNCTION Function, + IN gceTEXTURE_SOURCE Source0, + IN gceTEXTURE_CHANNEL Channel0, + IN gceTEXTURE_SOURCE Source1, + IN gceTEXTURE_CHANNEL Channel1, + IN gceTEXTURE_SOURCE Source2, + IN gceTEXTURE_CHANNEL Channel2, + IN gctINT Scale + ); + +/******************************************************************************\ +******************************* gcoTEXTURE Object ******************************* +\******************************************************************************/ + +typedef struct _gcsTEXTURE +{ + /* Addressing modes. */ + gceTEXTURE_ADDRESSING s; + gceTEXTURE_ADDRESSING t; + gceTEXTURE_ADDRESSING r; + + gceTEXTURE_SWIZZLE swizzle[gcvTEXTURE_COMPONENT_NUM]; + + /* Border color. */ + gctUINT8 border[gcvTEXTURE_COMPONENT_NUM]; + + /* Filters. */ + gceTEXTURE_FILTER minFilter; + gceTEXTURE_FILTER magFilter; + gceTEXTURE_FILTER mipFilter; + gctUINT anisoFilter; + + /* Level of detail. */ + gctFLOAT lodBias; + gctFLOAT lodMin; + gctFLOAT lodMax; + + /* base/max level */ + gctINT32 baseLevel; + gctINT32 maxLevel; + + /* depth texture comparison */ + gceTEXTURE_COMPARE_MODE compareMode; + gceCOMPARE compareFunc; + + gceTEXTURE_DS_TEX_MODE dsTextureMode; + + gceTEXTURE_DS_MODE dsMode; + + /* sRGB decode */ + gceTEXTURE_SRGBDECODE sRGB; + + gcuVALUE borderColor[4]; + gctBOOL descDirty; +} +gcsTEXTURE, * gcsTEXTURE_PTR; + +typedef struct _gcsTEXTURE_BINDTEXTS_ARGS +{ + /* must be the first member */ + gceHAL_ARG_VERSION version; + +} +gcsTEXTURE_BINDTEXTS_ARGS; + +/* Construct a new gcoTEXTURE object. */ +gceSTATUS +gcoTEXTURE_Construct( + IN gcoHAL Hal, + OUT gcoTEXTURE * Texture + ); + +/* Construct a new gcoTEXTURE object with type information. */ +gceSTATUS +gcoTEXTURE_ConstructEx( + IN gcoHAL Hal, + IN gceTEXTURE_TYPE Type, + OUT gcoTEXTURE * Texture + ); + + +/* Construct a new sized gcoTEXTURE object. */ +gceSTATUS +gcoTEXTURE_ConstructSized( + IN gcoHAL Hal, + IN gceSURF_FORMAT Format, + IN gctUINT Width, + IN gctUINT Height, + IN gctUINT Depth, + IN gctUINT Faces, + IN gctUINT MipMapCount, + IN gcePOOL Pool, + OUT gcoTEXTURE * Texture + ); + +/* Destroy an gcoTEXTURE object. */ +gceSTATUS +gcoTEXTURE_Destroy( + IN gcoTEXTURE Texture + ); + +/* Upload data to an gcoTEXTURE object. */ +gceSTATUS +gcoTEXTURE_Upload( + IN gcoTEXTURE Texture, + IN gctINT MipMap, + IN gceTEXTURE_FACE Face, + IN gctSIZE_T Width, + IN gctSIZE_T Height, + IN gctUINT Slice, + IN gctCONST_POINTER Memory, + IN gctSIZE_T Stride, + IN gceSURF_FORMAT Format, + IN gceSURF_COLOR_SPACE SrcColorSpace + ); + +/* Upload data to an gcoTEXTURE object. */ +gceSTATUS +gcoTEXTURE_UploadSub( + IN gcoTEXTURE Texture, + IN gctINT MipMap, + IN gceTEXTURE_FACE Face, + IN gctSIZE_T X, + IN gctSIZE_T Y, + IN gctSIZE_T Width, + IN gctSIZE_T Height, + IN gctUINT Slice, + IN gctCONST_POINTER Memory, + IN gctSIZE_T Stride, + IN gceSURF_FORMAT Format, + IN gceSURF_COLOR_SPACE SrcColorSpace, + IN gctUINT32 PhysicalAddress + ); + + +/* Upload YUV data to an gcoTEXTURE object. */ +gceSTATUS +gcoTEXTURE_UploadYUV( + IN gcoTEXTURE Texture, + IN gceTEXTURE_FACE Face, + IN gctUINT Width, + IN gctUINT Height, + IN gctUINT Slice, + IN gctPOINTER Memory[3], + IN gctINT Stride[3], + IN gceSURF_FORMAT Format + ); + +/* Upload compressed data to an gcoTEXTURE object. */ +gceSTATUS +gcoTEXTURE_UploadCompressed( + IN gcoTEXTURE Texture, + IN gctINT MipMap, + IN gceTEXTURE_FACE Face, + IN gctSIZE_T Width, + IN gctSIZE_T Height, + IN gctUINT Slice, + IN gctCONST_POINTER Memory, + IN gctSIZE_T Bytes + ); + +/* Upload compressed sub data to an gcoTEXTURE object. */ +gceSTATUS +gcoTEXTURE_UploadCompressedSub( + IN gcoTEXTURE Texture, + IN gctINT MipMap, + IN gceTEXTURE_FACE Face, + IN gctSIZE_T XOffset, + IN gctSIZE_T YOffset, + IN gctSIZE_T Width, + IN gctSIZE_T Height, + IN gctUINT Slice, + IN gctCONST_POINTER Memory, + IN gctSIZE_T Size + ); + +/* Get gcoSURF object for a mipmap level. */ +gceSTATUS +gcoTEXTURE_GetMipMap( + IN gcoTEXTURE Texture, + IN gctUINT MipMap, + OUT gcoSURF * Surface + ); + +/* Get gcoSURF object for a mipmap level and face offset. */ +gceSTATUS +gcoTEXTURE_GetMipMapFace( + IN gcoTEXTURE Texture, + IN gctUINT MipMap, + IN gceTEXTURE_FACE Face, + OUT gcoSURF * Surface, + OUT gctSIZE_T_PTR Offset + ); + +gceSTATUS +gcoTEXTURE_GetMipMapSlice( + IN gcoTEXTURE Texture, + IN gctUINT MipMap, + IN gctUINT Slice, + OUT gcoSURF * Surface, + OUT gctSIZE_T_PTR Offset + ); + +gceSTATUS +gcoTEXTURE_AddMipMap( + IN gcoTEXTURE Texture, + IN gctINT Level, + IN gctINT InternalFormat, + IN gceSURF_FORMAT Format, + IN gctSIZE_T Width, + IN gctSIZE_T Height, + IN gctSIZE_T Depth, + IN gctUINT Faces, + IN gcePOOL Pool, + IN gctBOOL Filterable, + OUT gcoSURF * Surface + ); + +gceSTATUS +gcoTEXTURE_AddMipMapEx( + IN gcoTEXTURE Texture, + IN gctINT Level, + IN gctINT InternalFormat, + IN gceSURF_FORMAT Format, + IN gctSIZE_T Width, + IN gctSIZE_T Height, + IN gctSIZE_T Depth, + IN gctUINT Faces, + IN gcePOOL Pool, + IN gctUINT32 Samples, + IN gctBOOL Protected, + IN gctBOOL Filterable, + OUT gcoSURF * Surface + ); + +gceSTATUS +gcoTEXTURE_AddMipMapFromClient( + IN gcoTEXTURE Texture, + IN gctINT Level, + IN gcoSURF Surface + ); + +gceSTATUS +gcoTEXTURE_AddMipMapFromSurface( + IN gcoTEXTURE Texture, + IN gctINT Level, + IN gcoSURF Surface + ); + +gceSTATUS +gcoTEXTURE_LockMipMap( + IN gcoTEXTURE Texture, + IN gctUINT MipMap, + OPTIONAL OUT gctUINT32 * Address, + OPTIONAL OUT gctPOINTER * Memory + ); + +gceSTATUS +gcoTEXTURE_SetEndianHint( + IN gcoTEXTURE Texture, + IN gceENDIAN_HINT EndianHint + ); + +gceSTATUS +gcoTEXTURE_Disable( + IN gcoHAL Hal, + IN gctINT Sampler, + IN gctBOOL DefaultInteger + ); + +gceSTATUS +gcoTEXTURE_Flush( + IN gcoTEXTURE Texture + ); + +gceSTATUS +gcoTEXTURE_FlushVS( + IN gcoTEXTURE Texture + ); + +gceSTATUS +gcoTEXTURE_QueryCaps( + IN gcoHAL Hal, + OUT gctUINT * MaxWidth, + OUT gctUINT * MaxHeight, + OUT gctUINT * MaxDepth, + OUT gctBOOL * Cubic, + OUT gctBOOL * NonPowerOfTwo, + OUT gctUINT * VertexSamplers, + OUT gctUINT * PixelSamplers + ); + +gceSTATUS +gcoTEXTURE_GetClosestFormat( + IN gcoHAL Hal, + IN gceSURF_FORMAT InFormat, + OUT gceSURF_FORMAT* OutFormat + ); + +gceSTATUS +gcoTEXTURE_GetClosestFormatEx( + IN gcoHAL Hal, + IN gceSURF_FORMAT InFormat, + IN gceTEXTURE_TYPE TextureType, + OUT gceSURF_FORMAT* OutFormat + ); + +gceSTATUS +gcoTEXTURE_GetFormatInfo( + IN gcoTEXTURE Texture, + IN gctINT preferLevel, + OUT gcsSURF_FORMAT_INFO_PTR * TxFormatInfo + ); + +gceSTATUS +gcoTEXTURE_GetTextureFormatName( + IN gcsSURF_FORMAT_INFO_PTR TxFormatInfo, + OUT gctCONST_STRING * TxName + ); + +gceSTATUS +gcoTEXTURE_RenderIntoMipMap( + IN gcoTEXTURE Texture, + IN gctINT Level + ); + +gceSTATUS +gcoTEXTURE_RenderIntoMipMap2( + IN gcoTEXTURE Texture, + IN gctINT Level, + IN gctBOOL Sync + ); + +gceSTATUS +gcoTEXTURE_IsRenderable( + IN gcoTEXTURE Texture, + IN gctUINT Level + ); + +gceSTATUS +gcoTEXTURE_IsComplete( + IN gcoTEXTURE Texture, + IN gcsTEXTURE_PTR Info, + IN gctINT BaseLevel, + IN gctINT MaxLevel + ); + +gceSTATUS +gcoTEXTURE_CheckTexLevel0Attrib( + IN gcoTEXTURE Texture, + IN gctINT MaxLevel, + IN gctINT usedLevel + ); + +gceSTATUS +gcoTEXTURE_BindTexture( + IN gcoTEXTURE Texture, + IN gctINT Target, + IN gctINT Sampler, + IN gcsTEXTURE_PTR Info + ); + +gceSTATUS +gcoTEXTURE_BindTextureEx( + IN gcoTEXTURE Texture, + IN gctINT Target, + IN gctINT Sampler, + IN gcsTEXTURE_PTR Info, + IN gctINT textureLayer + ); + +gceSTATUS +gcoTEXTURE_BindTextureDesc( + IN gcoTEXTURE Texture, + IN gctINT Sampler, + IN gcsTEXTURE_PTR Info, + IN gctINT TextureLayer + ); + +gceSTATUS +gcoTEXTURE_SetDescDirty( + IN gcoTEXTURE Texture + ); + +gceSTATUS +gcoTEXTURE_InitParams( + IN gcoHAL Hal, + IN gcsTEXTURE_PTR TexParams + ); + +gceSTATUS +gcoTEXTURE_SetDepthTextureFlag( + IN gcoTEXTURE Texture, + IN gctBOOL unsized + ); + +gceSTATUS +gcoTEXTURE_BindTextureTS( + IN gcsTEXTURE_BINDTEXTS_ARGS * args + ); + +gceSTATUS +gcoTEXTURE_GenerateMipMap( + IN gcoTEXTURE Texture, + IN gctINT BaseLevel, + IN gctINT MaxLevel, + IN gctBOOL sRGBDecode + ); + +/******************************************************************************\ +******************************* gcoSTREAM Object ****************************** +\******************************************************************************/ + +gceSTATUS +gcoSTREAM_Construct( + IN gcoHAL Hal, + OUT gcoSTREAM * Stream + ); + +gceSTATUS +gcoSTREAM_Destroy( + IN gcoSTREAM Stream + ); + +gceSTATUS +gcoSTREAM_Upload( + IN gcoSTREAM Stream, + IN gctCONST_POINTER Buffer, + IN gctSIZE_T Offset, + IN gctSIZE_T Bytes, + IN gctBOOL Dynamic + ); + +gceSTATUS +gcoSTREAM_ReAllocBufNode( + IN gcoSTREAM Stream + ); + +gceSTATUS +gcoSTREAM_SetStride( + IN gcoSTREAM Stream, + IN gctUINT32 Stride + ); + +gceSTATUS +gcoSTREAM_Node( + IN gcoSTREAM Stream, + OUT gcsSURF_NODE_PTR * Node + ); + +gceSTATUS +gcoSTREAM_Lock( + IN gcoSTREAM Stream, + OUT gctPOINTER * Logical, + OUT gctUINT32 * Physical + ); + +gceSTATUS +gcoSTREAM_Unlock( + IN gcoSTREAM Stream + ); + +gceSTATUS +gcoSTREAM_Reserve( + IN gcoSTREAM Stream, + IN gctSIZE_T Bytes + ); + +gceSTATUS +gcoSTREAM_Flush( + IN gcoSTREAM Stream + ); + +typedef struct _gcsSTREAM_INFO +{ + gctUINT index; + gceVERTEX_FORMAT format; + gctBOOL normalized; + gctUINT components; + gctSIZE_T size; + gctCONST_POINTER data; + gctUINT stride; +} +gcsSTREAM_INFO, * gcsSTREAM_INFO_PTR; + +gceSTATUS +gcoSTREAM_CPUCacheOperation( + IN gcoSTREAM Stream, + IN gceCACHEOPERATION Operation + ); + +gceSTATUS +gcoSTREAM_CPUCacheOperation_Range( + IN gcoSTREAM Stream, + IN gctSIZE_T Offset, + IN gctSIZE_T Length, + IN gceCACHEOPERATION Operation + ); + +/******************************************************************************\ +******************************** gcoVERTEX Object ****************************** +\******************************************************************************/ + +typedef struct _gcsVERTEX_ATTRIBUTES +{ + gceVERTEX_FORMAT format; + gctBOOL normalized; + gctUINT32 components; + gctSIZE_T size; + gctUINT32 stream; + gctUINT32 offset; + gctUINT32 stride; +} +gcsVERTEX_ATTRIBUTES; + +gceSTATUS +gcoVERTEX_Construct( + IN gcoHAL Hal, + OUT gcoVERTEX * Vertex + ); + +gceSTATUS +gcoVERTEX_Destroy( + IN gcoVERTEX Vertex + ); + +gceSTATUS +gcoVERTEX_Reset( + IN gcoVERTEX Vertex + ); + +gceSTATUS +gcoVERTEX_EnableAttribute( + IN gcoVERTEX Vertex, + IN gctUINT32 Index, + IN gceVERTEX_FORMAT Format, + IN gctBOOL Normalized, + IN gctUINT32 Components, + IN gcoSTREAM Stream, + IN gctUINT32 Offset, + IN gctUINT32 Stride + ); + +gceSTATUS +gcoVERTEX_DisableAttribute( + IN gcoVERTEX Vertex, + IN gctUINT32 Index + ); + +gceSTATUS +gcoVERTEX_Bind( + IN gcoVERTEX Vertex + ); + +/******************************************************************************* +***** gcoVERTEXARRAY Object ***************************************************/ + +typedef struct _gcsATTRIBUTE +{ + /* Enabled. */ + gctBOOL enable; + + /* Number of components. */ + gctINT size; + + /* Attribute format. */ + gceVERTEX_FORMAT format; + + /* Flag whether the attribute is normalized or not. */ + gctBOOL normalized; + + /* Stride of the component. */ + gctSIZE_T stride; + + /* Divisor of the attribute */ + gctUINT divisor; + + /* Pointer to the attribute data. */ + gctCONST_POINTER pointer; + + /* Stream object owning the attribute data. */ + gcoBUFOBJ stream; + + /* Generic values for attribute. */ + gctFLOAT genericValue[4]; + + /* Generic size for attribute. */ + gctINT genericSize; + + /* Vertex shader linkage. */ + gctUINT linkage; + +#if gcdUSE_WCLIP_PATCH + /* Does it hold positions? */ + gctBOOL isPosition; +#endif + + /* Index to vertex array */ + gctINT arrayIdx; + + gceATTRIB_SCHEME convertScheme; + + /* Pointer to the temporary buffer to be freed */ + gcoBUFOBJ tempStream; + + /* Pointer to the temporary memory to be freed */ + gctCONST_POINTER tempMemory; +} +gcsATTRIBUTE, +* gcsATTRIBUTE_PTR; + +typedef struct _gcsVERTEXARRAY +{ + /* Enabled. */ + gctBOOL enable; + + /* Number of components. */ + gctINT size; + + /* Attribute format. */ + gceVERTEX_FORMAT format; + + /* Flag whether the attribute is normalized or not. */ + gctBOOL normalized; + + /* Stride of the component. */ + gctUINT stride; + + /* Divisor of the attribute */ + gctUINT divisor; + + /* Pointer to the attribute data. */ + gctCONST_POINTER pointer; + + /* Stream object owning the attribute data. */ + gcoSTREAM stream; + + /* Generic values for attribute. */ + gctFLOAT genericValue[4]; + + /* Generic size for attribute. */ + gctINT genericSize; + + /* Vertex shader linkage. */ + gctUINT linkage; + + gctBOOL isPosition; +} +gcsVERTEXARRAY, +* gcsVERTEXARRAY_PTR; + +gceSTATUS +gcoVERTEXARRAY_Construct( + IN gcoHAL Hal, + OUT gcoVERTEXARRAY * Vertex + ); + +gceSTATUS +gcoVERTEXARRAY_Destroy( + IN gcoVERTEXARRAY Vertex + ); + +/* If don't consider isolation, STREAM_INFO / INDEX_INFO could be +** include in the struct of instantDraw in chip level.*/ +typedef struct _gcsVERTEXARRAY_STREAM_INFO +{ + gctUINT attribMask; + gctSIZE_T first; + gctSIZE_T count; + gcePRIMITIVE primMode; + gctSIZE_T primCount; + gctINT vertexInstIndex; + gctBOOL instanced; + gctSIZE_T instanceCount; + + union _gcsVERTEXARRAY_STREAM_INFO_UNION + { + struct _gcsVERTEXARRAY_STREAM_ES11_INFO + { + gcsVERTEXARRAY_PTR attributes; + }es11; + + struct _gcsVERTEXARRAY_STREAM_ES30_INFO + { + gcsATTRIBUTE_PTR attributes; + }es30; + }u; +}gcsVERTEXARRAY_STREAM_INFO, +*gcsVERTEXARRAY_STREAM_INFO_PTR; + +typedef const struct _gcsVERTEXARRAY_STREAM_INFO* gcsVERTEXARRAY_STREAM_INFO_CONST_PTR; + +typedef struct _gcsVERTEXARRAY_INDEX_INFO +{ + gctSIZE_T count; + gceINDEX_TYPE indexType; + gctPOINTER indexMemory; + gctUINT restartElement; + + union _gcsVERTEXARRAY_INDEX_INFO_UNION + { + struct _gcsVERTEXARRAY_INDEX_ES11_INFO + { + gcoINDEX indexBuffer; + }es11; + + struct _gcsVERTEXARRAY_INDEX_ES30_INFO + { + gcoBUFOBJ indexBuffer; + }es30; + }u; +}gcsVERTEXARRAY_INDEX_INFO, +*gcsVERTEXARRAY_INDEX_INFO_PTR; + +typedef const struct _gcsVERTEXARRAY_INDEX_INFO* gcsVERTEXARRAY_INDEX_INFO_CONST_PTR; + +gceSTATUS +gcoVERTEXARRAY_IndexBind( + IN gcoVERTEXARRAY Vertex, + IN gcsVERTEXARRAY_INDEX_INFO_PTR IndexInfo + ); + +gceSTATUS +gcoVERTEXARRAY_StreamBind( + IN gcoVERTEXARRAY Vertex, +#if gcdUSE_WCLIP_PATCH + IN OUT gctFLOAT * WLimitRms, + IN OUT gctBOOL * WLimitRmsDirty, +#endif + IN gcsVERTEXARRAY_STREAM_INFO_CONST_PTR StreamInfo, + IN gcsVERTEXARRAY_INDEX_INFO_CONST_PTR IndexInfo + ); + +gceSTATUS +gcoVERTEXARRAY_IndexBind_Ex( + IN gcoVERTEXARRAY Vertex, + IN OUT gcsVERTEXARRAY_STREAM_INFO_PTR StreamInfo, + IN gcsVERTEXARRAY_INDEX_INFO_PTR IndexInfo + ); + +gceSTATUS +gcoVERTEXARRAY_StreamBind_Ex( + IN gcoVERTEXARRAY Vertex, +#if gcdUSE_WCLIP_PATCH + IN OUT gctFLOAT * WLimitRms, + IN OUT gctBOOL * WLimitRmsDirty, +#endif + IN OUT gcsVERTEXARRAY_STREAM_INFO_PTR StreamInfo, + IN gcsVERTEXARRAY_INDEX_INFO_PTR IndexInfo + ); + +gceSTATUS +gcoVERTEXARRAY_Bind( + IN gcoVERTEXARRAY Vertex, + IN gctUINT32 EnableBits, + IN gcsVERTEXARRAY_PTR VertexArray, + IN gctUINT First, + IN gctSIZE_T * Count, + IN gceINDEX_TYPE IndexType, + IN gcoINDEX IndexObject, + IN gctPOINTER IndexMemory, + IN OUT gcePRIMITIVE * PrimitiveType, +#if gcdUSE_WCLIP_PATCH + IN OUT gctUINT * PrimitiveCount, + IN OUT gctFLOAT * wLimitRms, + IN OUT gctBOOL * wLimitDirty +#else + IN OUT gctUINT * PrimitiveCount +#endif + ); + +/* Frame Database */ +gceSTATUS +gcoHAL_AddFrameDB( + void + ); + +gceSTATUS +gcoHAL_DumpFrameDB( + gctCONST_STRING Filename OPTIONAL + ); + +gceSTATUS +gcoHAL_InitGPUProfile( + void + ); + +gceSTATUS +gcoHAL_DumpGPUProfile( + void + ); + +/****************************************************************************** +**********************gcoBUFOBJ object***************************************** +*******************************************************************************/ +/* Construct a new gcoBUFOBJ object. */ +gceSTATUS +gcoBUFOBJ_Construct( + IN gcoHAL Hal, + IN gceBUFOBJ_TYPE Type, + OUT gcoBUFOBJ * BufObj + ); + +/* Destroy a gcoBUFOBJ object. */ +gceSTATUS +gcoBUFOBJ_Destroy( + IN gcoBUFOBJ BufObj + ); + +/* Lock pbo in memory. */ +gceSTATUS +gcoBUFOBJ_Lock( + IN gcoBUFOBJ BufObj, + OUT gctUINT32 * Address, + OUT gctPOINTER * Memory + ); + +/* Lock pbo in memory. */ +gceSTATUS +gcoBUFOBJ_FastLock( + IN gcoBUFOBJ BufObj, + OUT gctUINT32 * Address, + OUT gctPOINTER * Memory + ); + +/* Unlock pbo that was previously locked with gcoBUFOBJ_Lock. */ +gceSTATUS +gcoBUFOBJ_Unlock( + IN gcoBUFOBJ BufObj + ); + +/* Free existing pbo buffer. */ +gceSTATUS +gcoBUFOBJ_Free( + IN gcoBUFOBJ BufObj + ); + +/* Upload data into an pbo buffer. */ +gceSTATUS +gcoBUFOBJ_Upload( + IN gcoBUFOBJ BufObj, + IN gctCONST_POINTER Buffer, + IN gctSIZE_T Offset, + IN gctSIZE_T Bytes, + IN gceBUFOBJ_USAGE Usage + ); + +/* Bind an index object to the hardware. */ +gceSTATUS +gcoBUFOBJ_IndexBind ( + IN gcoBUFOBJ Index, + IN gceINDEX_TYPE Type, + IN gctSIZE_T Offset, + IN gctSIZE_T Count, + IN gctUINT RestartElement + ); + +/* Find min and max index for the index buffer */ +gceSTATUS +gcoBUFOBJ_IndexGetRange( + IN gcoBUFOBJ Index, + IN gceINDEX_TYPE Type, + IN gctSIZE_T Offset, + IN gctUINT32 Count, + OUT gctUINT32 * MinimumIndex, + OUT gctUINT32 * MaximumIndex + ); + +/* Sets a buffer object as dirty */ +gceSTATUS +gcoBUFOBJ_SetDirty( + IN gcoBUFOBJ BufObj + ); + +/* Creates a new buffer if needed */ +gceSTATUS +gcoBUFOBJ_AlignIndexBufferWhenNeeded( + IN gcoBUFOBJ BufObj, + IN gctSIZE_T Offset, + OUT gcoBUFOBJ * AlignedBufObj + ); + +/* Cache operations on whole range */ +gceSTATUS +gcoBUFOBJ_CPUCacheOperation( + IN gcoBUFOBJ BufObj, + IN gceCACHEOPERATION Operation + ); + +/* Cache operations on a specified range */ +gceSTATUS +gcoBUFOBJ_CPUCacheOperation_Range( + IN gcoBUFOBJ BufObj, + IN gctSIZE_T Offset, + IN gctSIZE_T Length, + IN gceCACHEOPERATION Operation + ); + +/* Return size of the bufobj */ +gceSTATUS +gcoBUFOBJ_GetSize( + IN gcoBUFOBJ BufObj, + OUT gctSIZE_T_PTR Size + ); + +/* Return memory node of the bufobj */ +gceSTATUS +gcoBUFOBJ_GetNode( + IN gcoBUFOBJ BufObj, + OUT gcsSURF_NODE_PTR * Node + ); + +gceSTATUS +gcoBUFOBJ_ReAllocBufNode( + IN gcoBUFOBJ BufObj + ); + +/* Handle GPU cache operations */ +gceSTATUS +gcoBUFOBJ_SetCPUWrite( + gcoBUFOBJ BufObj, + gctBOOL Value + ); + +/* Dump buffer. */ +void +gcoBUFOBJ_Dump( + IN gcoBUFOBJ BufObj + ); + +#endif /* gcdENABLE_3D */ + +#ifdef __cplusplus +} +#endif + +#endif /* __gc_hal_engine_h_ */ + +
diff --git a/hal/kernel/inc/gc_hal_engine_vg.h b/hal/kernel/inc/gc_hal_engine_vg.h new file mode 100644 index 0000000..2558725 --- /dev/null +++ b/hal/kernel/inc/gc_hal_engine_vg.h
@@ -0,0 +1,1059 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_engine_vg_h_ +#define __gc_hal_engine_vg_h_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "shared/gc_hal_types.h" + +/******************************************************************************\ +******************************** VG Structures ******************************* +\******************************************************************************/ + +/** +** @ingroup gcoVG +** +** @brief Definition of the color ramp used by the gradient paints. +** +** The gcsCOLOR_RAMP structure defines the layout of one single color inside +** a color ramp which is used by gradient paints. +*/ +typedef struct _gcsCOLOR_RAMP +{ + /** Value for the color stop. */ + gctFLOAT stop; + + /** Red color channel value for the color stop. */ + gctFLOAT red; + + /** Green color channel value for the color stop. */ + gctFLOAT green; + + /** Blue color channel value for the color stop. */ + gctFLOAT blue; + + /** Alpha color channel value for the color stop. */ + gctFLOAT alpha; +} +gcsCOLOR_RAMP, * gcsCOLOR_RAMP_PTR; + +/** +** @ingroup gcoVG +** +** @brief Definition of the color ramp used by the gradient paints in fixed form. +** +** The gcsCOLOR_RAMP structure defines the layout of one single color inside +** a color ramp which is used by gradient paints. +*/ +typedef struct _gcsFIXED_COLOR_RAMP +{ + /** Value for the color stop. */ + gctFIXED_POINT stop; + + /** Red color channel value for the color stop. */ + gctFIXED_POINT red; + + /** Green color channel value for the color stop. */ + gctFIXED_POINT green; + + /** Blue color channel value for the color stop. */ + gctFIXED_POINT blue; + + /** Alpha color channel value for the color stop. */ + gctFIXED_POINT alpha; +} +gcsFIXED_COLOR_RAMP, * gcsFIXED_COLOR_RAMP_PTR; + + +/** +** @ingroup gcoVG +** +** @brief Rectangle structure used by the gcoVG object. +** +** This structure defines the layout of a rectangle. Make sure width and +** height are larger than 0. +*/ +typedef struct _gcsVG_RECT * gcsVG_RECT_PTR; +typedef struct _gcsVG_RECT +{ + /** Left location of the rectangle. */ + gctINT x; + + /** Top location of the rectangle. */ + gctINT y; + + /** Width of the rectangle. */ + gctINT width; + + /** Height of the rectangle. */ + gctINT height; +} +gcsVG_RECT; + +/** +** @ingroup gcoVG +** +** @brief Path command buffer attribute structure. +** +** The gcsPATH_BUFFER_INFO structure contains the specifics about +** the layout of the path data command buffer. +*/ +typedef struct _gcsPATH_BUFFER_INFO * gcsPATH_BUFFER_INFO_PTR; +typedef struct _gcsPATH_BUFFER_INFO +{ + gctUINT reservedForHead; + gctUINT reservedForTail; +} +gcsPATH_BUFFER_INFO; + +/** +** @ingroup gcoVG +** +** @brief Definition of the path data container structure. +** +** The gcsPATH structure defines the layout of the path data container. +*/ +typedef struct _gcsPATH_DATA * gcsPATH_DATA_PTR; +typedef struct _gcsPATH_DATA +{ + /* Data container in command buffer format. */ + gcsCMDBUFFER data; + + /* Path data type. */ + gcePATHTYPE dataType; +} +gcsPATH_DATA; + + +/******************************************************************************\ +********************************* gcoHAL Object ******************************** +\******************************************************************************/ + +/* Query path data storage attributes. */ +gceSTATUS +gcoHAL_QueryPathStorage( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + OUT gcsPATH_BUFFER_INFO_PTR Information + ); + +/* Associate a completion signal with the command buffer. */ +gceSTATUS +gcoHAL_AssociateCompletion( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcsPATH_DATA_PTR PathData + ); + +/* Release the current command buffer completion signal. */ +gceSTATUS +gcoHAL_DeassociateCompletion( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcsPATH_DATA_PTR PathData + ); + +/* Verify whether the command buffer is still in use. */ +gceSTATUS +gcoHAL_CheckCompletion( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcsPATH_DATA_PTR PathData + ); + +/* Wait until the command buffer is no longer in use. */ +gceSTATUS +gcoHAL_WaitCompletion( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcsPATH_DATA_PTR PathData + ); + +/* Flush the pixel cache. */ +gceSTATUS +gcoHAL_Flush( +#if gcdGC355_PROFILER + IN gcoHAL Hal, + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth +#else + IN gcoHAL Hal +#endif + ); + +/* Split a harwdare address into pool and offset. */ +gceSTATUS +gcoHAL_SplitAddress( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctUINT32 Address, + OUT gcePOOL * Pool, + OUT gctUINT32 * Offset + ); + +/* Combine pool and offset into a harwdare address. */ +gceSTATUS +gcoHAL_CombineAddress( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcePOOL Pool, + IN gctUINT32 Offset, + OUT gctUINT32 * Address + ); + +/* Schedule to free linear video memory allocated. */ +gceSTATUS +gcoHAL_ScheduleVideoMemory( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctUINT32 Node + ); + +/* Free linear video memory allocated with gcoHAL_AllocateLinearVideoMemory. */ +gceSTATUS +gcoHAL_FreeVideoMemory( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctUINT32 Node, + IN gctBOOL asynchroneous + ); + +/* Query command buffer attributes. */ +gceSTATUS +gcoHAL_QueryCommandBuffer( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + OUT gcsCOMMAND_BUFFER_INFO_PTR Information + ); +/* Allocate and lock linear video memory. */ +gceSTATUS +gcoHAL_AllocateLinearVideoMemory( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctUINT Size, + IN gctUINT Alignment, + IN gcePOOL Pool, + OUT gctUINT32 * Node, + OUT gctUINT32 * Address, + OUT gctPOINTER * Memory + ); + +/* Align the specified size accordingly to the hardware requirements. */ +gceSTATUS +gcoHAL_GetAlignedSurfaceSize( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceSURF_TYPE Type, + IN OUT gctUINT32_PTR Width, + IN OUT gctUINT32_PTR Height + ); + +gceSTATUS +gcoHAL_ReserveTask( + IN gcoHAL Hal, +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceBLOCK Block, + IN gctUINT TaskCount, + IN gctUINT32 Bytes, + OUT gctPOINTER * Memory + ); +/******************************************************************************\ +********************************** gcoVG Object ******************************** +\******************************************************************************/ + +/** @defgroup gcoVG gcoVG +** +** The gcoVG object abstracts the VG hardware pipe. +*/ +#if gcdGC355_PROFILER +void +gcoVG_ProfilerEnableDisable( + IN gcoVG Vg, + IN gctUINT enableGetAPITimes, + IN gctFILE apiTimeFile + ); + +void +gcoVG_ProfilerTreeDepth( + IN gcoVG Vg, + IN gctUINT TreeDepth + ); + +void +gcoVG_ProfilerSetStates( + IN gcoVG Vg, + IN gctUINT treeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth + ); +#endif + +gctBOOL +gcoVG_IsMaskSupported( +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceSURF_FORMAT Format + ); + +gctBOOL +gcoVG_IsTargetSupported( +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceSURF_FORMAT Format + ); + +gctBOOL +gcoVG_IsImageSupported( +#if gcdGC355_PROFILER + IN gcoVG Vg, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceSURF_FORMAT Format + ); + +gctUINT8 gcoVG_PackColorComponent( +#if gcdGC355_PROFILER + gcoVG Vg, + gctUINT TreeDepth, + gctUINT saveLayerTreeDepth, + gctUINT varTreeDepth, +#endif + gctFLOAT Value + ); + +gceSTATUS +gcoVG_Construct( + IN gcoHAL Hal, + OUT gcoVG * Vg + ); + +gceSTATUS +gcoVG_Destroy( + IN gcoVG Vg +#if gcdGC355_PROFILER +, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth +#endif + ); + +void +gcoVG_SetTesselationSize( + IN gcoVG Vg, + IN gctUINT Width, + IN gctUINT Height + ); + +gceSTATUS +gcoVG_SetTarget( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Target, + IN gceORIENTATION orientation + ); + +gceSTATUS +gcoVG_UnsetTarget( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Surface + ); + +gceSTATUS +gcoVG_SetUserToSurface( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctFLOAT UserToSurface[9] + ); + +gceSTATUS +gcoVG_SetSurfaceToImage( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctFLOAT SurfaceToImage[9] + ); + +gceSTATUS +gcoVG_EnableMask( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctBOOL Enable + ); + +gceSTATUS +gcoVG_SetMask( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Mask + ); + +gceSTATUS +gcoVG_UnsetMask( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Surface + ); + +gceSTATUS +gcoVG_FlushMask( + IN gcoVG Vg +#if gcdGC355_PROFILER +, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth +#endif + ); + +gceSTATUS +gcoVG_EnableScissor( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctBOOL Enable + ); + +gceSTATUS +gcoVG_SetScissor( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctSIZE_T RectangleCount, + IN gcsVG_RECT_PTR Rectangles + ); + +gceSTATUS +gcoVG_EnableColorTransform( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctBOOL Enable + ); + +gceSTATUS +gcoVG_SetColorTransform( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctFLOAT ColorTransform[8] + ); + +gceSTATUS +gcoVG_SetTileFillColor( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctFLOAT Red, + IN gctFLOAT Green, + IN gctFLOAT Blue, + IN gctFLOAT Alpha + ); + +gceSTATUS +gcoVG_SetSolidPaint( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctUINT8 Red, + IN gctUINT8 Green, + IN gctUINT8 Blue, + IN gctUINT8 Alpha + ); + +gceSTATUS +gcoVG_SetLinearPaint( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctFLOAT Constant, + IN gctFLOAT StepX, + IN gctFLOAT StepY + ); + +gceSTATUS +gcoVG_SetRadialPaint( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctFLOAT LinConstant, + IN gctFLOAT LinStepX, + IN gctFLOAT LinStepY, + IN gctFLOAT RadConstant, + IN gctFLOAT RadStepX, + IN gctFLOAT RadStepY, + IN gctFLOAT RadStepXX, + IN gctFLOAT RadStepYY, + IN gctFLOAT RadStepXY + ); + +gceSTATUS +gcoVG_SetPatternPaint( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctFLOAT UConstant, + IN gctFLOAT UStepX, + IN gctFLOAT UStepY, + IN gctFLOAT VConstant, + IN gctFLOAT VStepX, + IN gctFLOAT VStepY, + IN gctBOOL Linear + ); + +gceSTATUS +gcoVG_SetColorRamp( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF ColorRamp, + IN gceTILE_MODE ColorRampSpreadMode + ); + +gceSTATUS +gcoVG_SetPattern( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctINT32 width, + IN gctINT32 height, + IN gcoSURF Pattern, + IN gceTILE_MODE TileMode, + IN gceIMAGE_FILTER Filter + ); + +gceSTATUS +gcoVG_SetImageMode( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceVG_IMAGE Mode + ); + +gceSTATUS +gcoVG_SetBlendMode( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceVG_BLEND Mode + ); + +gceSTATUS +gcoVG_SetRenderingQuality( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceRENDER_QUALITY Quality + ); + +gceSTATUS +gcoVG_SetFillRule( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceFILL_RULE FillRule + ); + +gceSTATUS +gcoVG_FinalizePath( + IN gcoVG Vg, + IN gcsPATH_DATA_PTR PathData + ); + +gceSTATUS +gcoVG_Clear( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctINT X, + IN gctINT Y, + IN gctINT Width, + IN gctINT Height + ); + +gceSTATUS +gcoVG_DrawPath( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcsPATH_DATA_PTR PathData, + IN gctFLOAT Scale, + IN gctFLOAT Bias, +#if gcdMOVG + IN gctUINT32 Width, + IN gctUINT32 Height, + IN gctFLOAT *Bounds, +#endif + IN gctBOOL SoftwareTesselation + ); + +gceSTATUS +gcoVG_DrawImage( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gceORIENTATION orientation, + IN gcoSURF Source, + IN gcsPOINT_PTR SourceOrigin, + IN gcsPOINT_PTR TargetOrigin, + IN gcsSIZE_PTR SourceSize, + IN gctINT SourceX, + IN gctINT SourceY, + IN gctINT TargetX, + IN gctINT TargetY, + IN gctINT Width, + IN gctINT Height, + IN gctBOOL Mask, + IN gctBOOL isDrawImage + ); + +gceSTATUS +gcoVG_TesselateImage( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Image, + IN gcsVG_RECT_PTR Rectangle, + IN gceIMAGE_FILTER Filter, + IN gctBOOL Mask, +#if gcdMOVG + IN gctBOOL SoftwareTesselation, + IN gceVG_BLEND BlendMode, + IN gctINT Width, + IN gctINT Height +#else + IN gctBOOL SoftwareTesselation +#endif + ); + +gceSTATUS +gcoVG_DrawSurfaceToImage( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Image, + IN const gcsVG_RECT_PTR SrcRectangle, + IN const gctFLOAT DstBounds[4], + IN const gctFLOAT DstPoints[8], + IN const gctFLOAT ImgMatrix[9], + IN const gctFLOAT RectMatrix[9], + IN gceIMAGE_FILTER Filter, + IN gctBOOL Mask, + IN gctBOOL FirstTime +#if gcdMOVG +, + IN gctINT TSWidth, + IN gctINT TSHeight +#endif + ); + +gceSTATUS +gcoVG_DrawSurfaceToImageMasked( + IN gcoVG Vg, + IN gcoSURF Image, + IN gcsVG_RECT_PTR SrcRect, + IN gctINT X, + IN gctINT Y, + IN gctINT Width, + IN gctINT Height, + IN const gctFLOAT Matrix[9] + ); + +gceSTATUS +gcoVG_Blit( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Source, + IN gcoSURF Target, + IN gcsVG_RECT_PTR SrcRect, + IN gcsVG_RECT_PTR TrgRect, + IN gceIMAGE_FILTER Filter, + IN gceVG_BLEND Mode + ); + +gceSTATUS +gcoVG_ColorMatrix( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Source, + IN gcoSURF Target, + IN const gctFLOAT * Matrix, + IN gceCHANNEL ColorChannels, + IN gctBOOL FilterLinear, + IN gctBOOL FilterPremultiplied, + IN gcsPOINT_PTR SourceOrigin, + IN gcsPOINT_PTR TargetOrigin, + IN gctINT Width, + IN gctINT Height + ); + +gceSTATUS +gcoVG_SeparableConvolve( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Source, + IN gcoSURF Target, + IN gctINT KernelWidth, + IN gctINT KernelHeight, + IN gctINT ShiftX, + IN gctINT ShiftY, + IN const gctINT16 * KernelX, + IN const gctINT16 * KernelY, + IN gctFLOAT Scale, + IN gctFLOAT Bias, + IN gceTILE_MODE TilingMode, + IN gctFLOAT_PTR FillColor, + IN gceCHANNEL ColorChannels, + IN gctBOOL FilterLinear, + IN gctBOOL FilterPremultiplied, + IN gcsPOINT_PTR SourceOrigin, + IN gcsPOINT_PTR TargetOrigin, + IN gcsSIZE_PTR SourceSize, + IN gctINT Width, + IN gctINT Height + ); + +gceSTATUS +gcoVG_GaussianBlur( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gcoSURF Source, + IN gcoSURF Target, + IN gctFLOAT StdDeviationX, + IN gctFLOAT StdDeviationY, + IN gceTILE_MODE TilingMode, + IN gctFLOAT_PTR FillColor, + IN gceCHANNEL ColorChannels, + IN gctBOOL FilterLinear, + IN gctBOOL FilterPremultiplied, + IN gcsPOINT_PTR SourceOrigin, + IN gcsPOINT_PTR TargetOrigin, + IN gcsSIZE_PTR SourceSize, + IN gctINT Width, + IN gctINT Height + ); + +gceSTATUS +gcoVG_EnableDither( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctBOOL Enable + ); + +/* Color Key States. */ +gceSTATUS +gcoVG_SetColorKey( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gcsPROFILERFUNCNODE *DList, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctUINT32* Values, + IN gctBOOL * Enables +); + +/* Index Color States. */ +gceSTATUS +gcoVG_SetColorIndexTable( + IN gcoVG Vg, +#if gcdGC355_PROFILER + IN gcsPROFILERFUNCNODE *DList, + IN gctUINT TreeDepth, + IN gctUINT saveLayerTreeDepth, + IN gctUINT varTreeDepth, +#endif + IN gctUINT32* Values, + IN gctINT32 Count +); + +gceSTATUS +gcoVG_SetYUV2RGBStdCust( + IN gcoVG Vg, + IN gctBOOL YUV2RGBStdCust + ); + +gceSTATUS +gcoVG_SetYUV2RGB( + IN gcoVG Vg, + IN gctFLOAT *coef, + IN gctFLOAT *offset, + IN gctBOOL *cfg +); + +gceSTATUS +gcoVG_SetRGB2YUVParameters( + IN gcoVG Vg, + IN gctFLOAT *coef, + IN gctFLOAT *offset, + IN gctBOOL *cfg +); + +/* VG RS feature support: YUV format conversion. */ +gceSTATUS +gcoVG_Resolve( + IN gcoVG Vg, + IN gcoSURF Source, + IN gcoSURF Target, + IN gctINT SX, + IN gctINT SY, + IN gctINT DX, + IN gctINT DY, + IN gctINT Width, + IN gctINT Height, + IN gctINT Src_uv, + IN gctINT Src_standard, + IN gctINT Dst_uv, + IN gctINT Dst_standard, + IN gctINT Dst_alpha, + IN gctBOOL Dst_standard_cust +); +#ifdef __cplusplus +} +#endif + +#endif /* __gc_hal_vg_h_ */ + +
diff --git a/hal/kernel/inc/gc_hal_enum.h b/hal/kernel/inc/gc_hal_enum.h new file mode 100644 index 0000000..e24750b --- /dev/null +++ b/hal/kernel/inc/gc_hal_enum.h
@@ -0,0 +1,2186 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#ifndef __gc_hal_enum_h_ +#define __gc_hal_enum_h_ + +#include "gc_hal_options.h" +#include "shared/gc_hal_enum.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* dummy draw type.*/ +typedef enum _gceDUMMY_DRAW_TYPE +{ + gcvDUMMY_DRAW_INVALID = 0, + gcvDUMMY_DRAW_GC400, + gcvDUMMY_DRAW_V60, +} +gceDUMMY_DRAW_TYPE; + +/* Option Set*/ +typedef enum _gceOPTION +{ + /* HW setting. */ + gcvOPTION_PREFER_ZCONVERT_BYPASS = 0, + gcvOPTION_PREFER_TILED_DISPLAY_BUFFER = 1, + gcvOPTION_PREFER_GUARDBAND = 2, + gcvOPTION_PREFER_TPG_TRIVIALMODEL = 3, + gcvOPTION_PREFER_RA_DEPTH_WRITE = 4, + gcvOPTION_PREFER_USC_RECONFIG = 5, + gcvOPTION_PREFER_DISALBE_HZ = 6, + + /* SW options */ + gcvOPTION_HW_NULL = 50, + gcvOPTION_PRINT_OPTION = 51, + gcvOPTION_KERNEL_FENCE = 52, + gcvOPTION_ASYNC_PIPE = 53, + gcvOPTION_FBO_PREFER_MEM = 54, + gcvOPTION_GPU_TEX_UPLOAD = 55, + gcvOPTION_GPU_BUFOBJ_UPLOAD = 56, + gcvOPTION_NO_Y_INVERT = 60, + + /* OCL option */ + gcvOPTION_OCL_ASYNC_BLT = 200, + gcvOPTION_OCL_IN_THREAD, + gcvOPTION_COMPRESSION_DEC400, + gcvOPTION_OCL_VIR_SHADER, + gcvOPTION_OCL_USE_MULTI_DEVICES, + +#if gcdUSE_VX + /* OVX options that HAL could access */ + gcvOPTION_OVX_ENABLE_NN_ZDP3 = 500, + gcvOPTION_OVX_ENABLE_NN_ZDP6, + gcvOPTION_OVX_ENABLE_NN_STRIDE, + gcvOPTION_OVX_USE_MULTI_DEVICES, + gcvOPTION_OVX_ENABLE_NN_DDR_BURST_SIZE_256B, + gcvOPTION_OVX_ENABLE_NN_DDR_BURST_SIZE_64B, +#endif + /* Insert option above this comment only */ + gcvOPTION_COUNT /* Not a OPTION*/ +} +gceOPTION; + +typedef enum _gceFRAMEINFO +{ + /* Total frame count in one run */ + gcvFRAMEINFO_FRAME_NUM = 0, + /* Total draw count in current frame, including draw/compute */ + gcvFRAMEINFO_DRAW_NUM = 1, + /* Total compute count in current frame, subset of drawNum */ + gcvFRAMEINFO_COMPUTE_NUM = 2, + /* Total dual16 draw/compute count in current frame, subset of drawNum */ + gcvFRAMEINFO_DUAL16_NUM = 3, + /* Current programID is being set. only valid for ES20 driver right now */ + gcvFRAMEINFO_PROGRAM_ID = 4, + + gcvFRAMEINFO_COUNT, +} +gceFRAMEINFO; + +typedef enum _gceFRAMEINFO_OP +{ + gcvFRAMEINFO_OP_INC = 0, + gcvFRAMEINFO_OP_DEC = 1, + gcvFRAMEINFO_OP_ZERO = 2, + gcvFRAMEINFO_OP_GET = 3, + gcvFRAMEINFO_OP_SET = 4, + gcvFRAMEINFO_OP_COUNT, +} +gceFRAMEINFO_OP; + +typedef enum _gceSURF_USAGE +{ + gcvSURF_USAGE_UNKNOWN, + gcvSURF_USAGE_RESOLVE_AFTER_CPU, + gcvSURF_USAGE_RESOLVE_AFTER_3D +} +gceSURF_USAGE; + +typedef enum _gceSURF_COLOR_SPACE +{ + gcvSURF_COLOR_SPACE_UNKNOWN, + gcvSURF_COLOR_SPACE_LINEAR, + gcvSURF_COLOR_SPACE_NONLINEAR, +} +gceSURF_COLOR_SPACE; + +typedef enum _gceSURF_COLOR_TYPE +{ + gcvSURF_COLOR_UNKNOWN = 0, + gcvSURF_COLOR_LINEAR = 0x01, + gcvSURF_COLOR_ALPHA_PRE = 0x02, +} +gceSURF_COLOR_TYPE; + +/* Rotation. */ +typedef enum _gceSURF_ROTATION +{ + gcvSURF_0_DEGREE = 0, + gcvSURF_90_DEGREE, + gcvSURF_180_DEGREE, + gcvSURF_270_DEGREE, + gcvSURF_FLIP_X, + gcvSURF_FLIP_Y, + + gcvSURF_POST_FLIP_X = 0x40000000, + gcvSURF_POST_FLIP_Y = 0x80000000, +} +gceSURF_ROTATION; + +/* Surface flag */ +typedef enum _gceSURF_FLAG +{ + /* None flag */ + gcvSURF_FLAG_NONE = 0x0, + /* content is preserved after swap */ + gcvSURF_FLAG_CONTENT_PRESERVED = 0x1, + /* content is updated after swap*/ + gcvSURF_FLAG_CONTENT_UPDATED = 0x2, + /* content is y inverted */ + gcvSURF_FLAG_CONTENT_YINVERTED = 0x4, + /* surface has multiple nodes */ + gcvSURF_FLAG_MULTI_NODE = 0x8, + /* surface no need do dither when resovle*/ + gcvSURF_FLAG_DITHER_DISABLED = 0x10, +} +gceSURF_FLAG; + +typedef enum _gceMIPMAP_IMAGE_FORMAT +{ + gcvUNKNOWN_MIPMAP_IMAGE_FORMAT = -2 +} +gceMIPMAP_IMAGE_FORMAT; + +typedef enum _gceIMAGE_MEM_TYPE +{ + gcvIMAGE_MEM_DEFAULT, + gcvIMAGE_MEM_HOST_PTR, + gcvIMAGE_MEM_HOST_PTR_UNCACHED, + gcvIMAGE_MEM_HOST_PHY_PTR, + gcvIMAGE_MEM_HOST_PHY_PTR_UNCACHED, +} +gceIMAGE_MEM_TYPE; + +typedef enum _gceSURF_YUV_COLOR_SPACE +{ + gcvSURF_ITU_REC601, + gcvSURF_ITU_REC709, + gcvSURF_ITU_REC2020, +} +gceSURF_YUV_COLOR_SPACE; + +typedef enum _gceSURF_YUV_SAMPLE_RANGE +{ + gcvSURF_YUV_FULL_RANGE, + gcvSURF_YUV_NARROW_RANGE, +} +gceSURF_YUV_SAMPLE_RANGE; + +typedef enum _gceSURF_YUV_CHROMA_SITING +{ + gcvSURF_YUV_CHROMA_SITING_0, + gcvSURF_YUV_CHROMA_SITING_0_5, +} +gceSURF_YUV_CHROMA_SITING; + +typedef enum _gceSURF_INFO_TYPE +{ + gcvSURF_INFO_UNKNOWN = 0, + gcvSURF_INFO_LAYERSIZE = 1, + gcvSURF_INFO_SLICESIZE = 2, +} +gceSURF_INFO_TYPE; + +/* Format modifiers. */ +typedef enum _gceSURF_FORMAT_MODE +{ + gcvSURF_FORMAT_OCL = 0x80000000, + gcvSURF_FORMAT_PATCH_BORDER = 0x40000000, +} +gceSURF_FORMAT_MODE; + +/* Pixel swizzle modes. */ +typedef enum _gceSURF_SWIZZLE +{ + gcvSURF_NOSWIZZLE = 0, + gcvSURF_ARGB, + gcvSURF_ABGR, + gcvSURF_RGBA, + gcvSURF_BGRA +} +gceSURF_SWIZZLE; + +/* Transparency modes. */ +typedef enum _gceSURF_TRANSPARENCY +{ + /* Valid only for PE 1.0 */ + gcvSURF_OPAQUE = 0, + gcvSURF_SOURCE_MATCH, + gcvSURF_SOURCE_MASK, + gcvSURF_PATTERN_MASK, +} +gceSURF_TRANSPARENCY; + +/* Surface Alignment. */ +typedef enum _gceSURF_ALIGNMENT +{ + gcvSURF_FOUR = 0, + gcvSURF_SIXTEEN, + gcvSURF_SUPER_TILED, + gcvSURF_SPLIT_TILED, + gcvSURF_SPLIT_SUPER_TILED +} +gceSURF_ALIGNMENT; + +/* Surface Addressing. */ +typedef enum _gceSURF_ADDRESSING +{ + gcvSURF_NO_STRIDE_TILED = 0, + gcvSURF_NO_STRIDE_LINEAR, + gcvSURF_STRIDE_TILED, + gcvSURF_STRIDE_LINEAR +} +gceSURF_ADDRESSING; + +/* Transparency modes. */ +typedef enum _gce2D_TRANSPARENCY +{ + /* Valid only for PE 2.0 */ + gcv2D_OPAQUE = 0, + gcv2D_KEYED, + gcv2D_MASKED +} +gce2D_TRANSPARENCY; + +/* Mono packing modes. */ +typedef enum _gceSURF_MONOPACK +{ + gcvSURF_PACKED8 = 0, + gcvSURF_PACKED16, + gcvSURF_PACKED32, + gcvSURF_UNPACKED, +} +gceSURF_MONOPACK; + +/* Blending modes. */ +typedef enum _gceSURF_BLEND_MODE +{ + /* Porter-Duff blending modes. */ + /* Fsrc Fdst */ + gcvBLEND_CLEAR = 0, /* 0 0 */ + gcvBLEND_SRC, /* 1 0 */ + gcvBLEND_DST, /* 0 1 */ + gcvBLEND_SRC_OVER_DST, /* 1 1 - Asrc */ + gcvBLEND_DST_OVER_SRC, /* 1 - Adst 1 */ + gcvBLEND_SRC_IN_DST, /* Adst 0 */ + gcvBLEND_DST_IN_SRC, /* 0 Asrc */ + gcvBLEND_SRC_OUT_DST, /* 1 - Adst 0 */ + gcvBLEND_DST_OUT_SRC, /* 0 1 - Asrc */ + gcvBLEND_SRC_ATOP_DST, /* Adst 1 - Asrc */ + gcvBLEND_DST_ATOP_SRC, /* 1 - Adst Asrc */ + gcvBLEND_SRC_XOR_DST, /* 1 - Adst 1 - Asrc */ + + /* Special blending modes. */ + gcvBLEND_SET, /* DST = 1 */ + gcvBLEND_SUB /* DST = DST * (1 - SRC) */ +} +gceSURF_BLEND_MODE; + +/* Per-pixel alpha modes. */ +typedef enum _gceSURF_PIXEL_ALPHA_MODE +{ + gcvSURF_PIXEL_ALPHA_STRAIGHT = 0, + gcvSURF_PIXEL_ALPHA_INVERSED +} +gceSURF_PIXEL_ALPHA_MODE; + +/* Global alpha modes. */ +typedef enum _gceSURF_GLOBAL_ALPHA_MODE +{ + gcvSURF_GLOBAL_ALPHA_OFF = 0, + gcvSURF_GLOBAL_ALPHA_ON, + gcvSURF_GLOBAL_ALPHA_SCALE +} +gceSURF_GLOBAL_ALPHA_MODE; + +/* Color component modes for alpha blending. */ +typedef enum _gceSURF_PIXEL_COLOR_MODE +{ + gcvSURF_COLOR_STRAIGHT = 0, + gcvSURF_COLOR_MULTIPLY +} +gceSURF_PIXEL_COLOR_MODE; + +/* Color component modes for alpha blending. */ +typedef enum _gce2D_PIXEL_COLOR_MULTIPLY_MODE +{ + gcv2D_COLOR_MULTIPLY_DISABLE = 0, + gcv2D_COLOR_MULTIPLY_ENABLE +} +gce2D_PIXEL_COLOR_MULTIPLY_MODE; + +/* Color component modes for alpha blending. */ +typedef enum _gce2D_GLOBAL_COLOR_MULTIPLY_MODE +{ + gcv2D_GLOBAL_COLOR_MULTIPLY_DISABLE = 0, + gcv2D_GLOBAL_COLOR_MULTIPLY_ALPHA, + gcv2D_GLOBAL_COLOR_MULTIPLY_COLOR +}<