blob: bc156f4e99d035d54ffb61f606cd173d76825a3f [file] [log] [blame]
/****************************************************************************
*
* The MIT License (MIT)
*
* Copyright (c) 2014 - 2020 Vivante Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
*****************************************************************************
*
* The GPL License (GPL)
*
* Copyright (C) 2014 - 2020 Vivante Corporation
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*****************************************************************************
*
* Note: This software is released under dual MIT and GPL licenses. A
* recipient may use this file under the terms of either the MIT license or
* GPL License. If you wish to use only one license not the other, you can
* indicate your decision by deleting one of the above license notices in your
* version of this file.
*
*****************************************************************************/
#include "gc_hal_kernel_linux.h"
#include "gc_hal_dump.h"
#include <linux/pagemap.h>
#include <linux/seq_file.h>
#include <linux/mman.h>
#include <asm/atomic.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/irqflags.h>
#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
#include <linux/math64.h>
#endif
#include <linux/delay.h>
#include <linux/platform_device.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,27)
#include <linux/anon_inodes.h>
#endif
#if gcdLINUX_SYNC_FILE
# include <linux/file.h>
# include "gc_hal_kernel_sync.h"
#endif
#if defined(CONFIG_DMA_SHARED_BUFFER)
#include <linux/dma-buf.h>
#endif
#define _GC_OBJ_ZONE gcvZONE_OS
#include "gc_hal_kernel_allocator.h"
#define gcmkBUG_ON(x) \
do { \
if (unlikely(!!(x))) \
{ \
printk("[galcore]: BUG ON @ %s(%d)\n", __func__, __LINE__); \
dump_stack(); \
} \
} while (0)
/******************************************************************************\
******************************* Private Functions ******************************
\******************************************************************************/
static gctINT
_GetThreadID(
void
)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24)
return task_pid_vnr(current);
#else
return current->pid;
#endif
}
/* Must hold Mdl->mpasMutex before call this function. */
static inline PLINUX_MDL_MAP
_CreateMdlMap(
IN PLINUX_MDL Mdl,
IN gctINT ProcessID
)
{
PLINUX_MDL_MAP mdlMap = gcvNULL;
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Mdl=%p ProcessID=%d", Mdl, ProcessID);
mdlMap = (PLINUX_MDL_MAP)kmalloc(sizeof(struct _LINUX_MDL_MAP), GFP_KERNEL | gcdNOWARN);
if (mdlMap == gcvNULL)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
mdlMap->pid = ProcessID;
mdlMap->vmaAddr = gcvNULL;
mdlMap->count = 0;
list_add(&mdlMap->link, &Mdl->mapsHead);
OnError:
gcmkFOOTER_ARG("ret=%p", mdlMap);
return mdlMap;
}
/* Must hold Mdl->mpasMutex before call this function. */
static inline gceSTATUS
_DestroyMdlMap(
IN PLINUX_MDL Mdl,
IN PLINUX_MDL_MAP MdlMap
)
{
gcmkHEADER_ARG("Mdl=%p MdlMap=%p", Mdl, MdlMap);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(MdlMap != gcvNULL);
list_del(&MdlMap->link);
kfree(MdlMap);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/* Must hold Mdl->mpasMutex before call this function. */
extern PLINUX_MDL_MAP
FindMdlMap(
IN PLINUX_MDL Mdl,
IN gctINT ProcessID
)
{
PLINUX_MDL_MAP mdlMap = gcvNULL;
gcmkHEADER_ARG("Mdl=%p ProcessID=%d", Mdl, ProcessID);
if (Mdl)
{
PLINUX_MDL_MAP iter = gcvNULL;
list_for_each_entry(iter, &Mdl->mapsHead, link)
{
if (iter->pid == ProcessID)
{
mdlMap = iter;
break;
}
}
}
gcmkFOOTER_ARG("ret=%p", mdlMap);
return mdlMap;
}
static PLINUX_MDL
_CreateMdl(
IN gckOS Os
)
{
PLINUX_MDL mdl;
gcmkHEADER();
mdl = (PLINUX_MDL)kzalloc(sizeof(struct _LINUX_MDL), GFP_KERNEL | gcdNOWARN);
if (mdl)
{
mdl->os = Os;
atomic_set(&mdl->refs, 1);
mutex_init(&mdl->mapsMutex);
INIT_LIST_HEAD(&mdl->mapsHead);
}
gcmkFOOTER_ARG("%p", mdl);
return mdl;
}
static gceSTATUS
_DestroyMdl(
IN PLINUX_MDL Mdl
)
{
gcmkHEADER_ARG("Mdl=%p", Mdl);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(Mdl != gcvNULL);
if (atomic_dec_and_test(&Mdl->refs))
{
gckOS os = Mdl->os;
gckALLOCATOR allocator = Mdl->allocator;
PLINUX_MDL_MAP mdlMap, next;
/* Valid private means alloc/attach successfully */
if (Mdl->priv)
{
if (Mdl->addr)
{
gcmALLOCATOR_UnmapKernel(allocator, Mdl, Mdl->addr);
}
gcmALLOCATOR_Free(allocator, Mdl);
}
mutex_lock(&Mdl->mapsMutex);
list_for_each_entry_safe(mdlMap, next, &Mdl->mapsHead, link)
{
gcmkVERIFY_OK(_DestroyMdlMap(Mdl, mdlMap));
}
mutex_unlock(&Mdl->mapsMutex);
if (Mdl->link.next)
{
/* Remove the node from global list.. */
mutex_lock(&os->mdlMutex);
list_del(&Mdl->link);
mutex_unlock(&os->mdlMutex);
}
kfree(Mdl);
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
** Integer Id Management.
*/
gceSTATUS
_AllocateIntegerId(
IN gcsINTEGER_DB_PTR Database,
IN gctPOINTER KernelPointer,
OUT gctUINT32 *Id
)
{
int result;
gctINT next;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
idr_preload(GFP_KERNEL | gcdNOWARN);
spin_lock(&Database->lock);
next = (Database->curr + 1 <= 0) ? 1 : Database->curr + 1;
result = idr_alloc(&Database->idr, KernelPointer, next, 0, GFP_ATOMIC);
/* ID allocated should not be 0. */
gcmkASSERT(result != 0);
if (result > 0)
{
Database->curr = *Id = result;
}
spin_unlock(&Database->lock);
idr_preload_end();
if (result < 0)
{
return gcvSTATUS_OUT_OF_RESOURCES;
}
#else
again:
if (idr_pre_get(&Database->idr, GFP_KERNEL | gcdNOWARN) == 0)
{
return gcvSTATUS_OUT_OF_MEMORY;
}
spin_lock(&Database->lock);
next = (Database->curr + 1 <= 0) ? 1 : Database->curr + 1;
/* Try to get a id greater than 0. */
result = idr_get_new_above(&Database->idr, KernelPointer, next, Id);
if (!result)
{
Database->curr = *Id;
}
spin_unlock(&Database->lock);
if (result == -EAGAIN)
{
goto again;
}
if (result != 0)
{
return gcvSTATUS_OUT_OF_RESOURCES;
}
#endif
return gcvSTATUS_OK;
}
gceSTATUS
_QueryIntegerId(
IN gcsINTEGER_DB_PTR Database,
IN gctUINT32 Id,
OUT gctPOINTER * KernelPointer
)
{
gctPOINTER pointer;
spin_lock(&Database->lock);
pointer = idr_find(&Database->idr, Id);
spin_unlock(&Database->lock);
if (pointer)
{
*KernelPointer = pointer;
return gcvSTATUS_OK;
}
else
{
gcmkTRACE_ZONE(
gcvLEVEL_ERROR, gcvZONE_OS,
"%s(%d) Id = %d is not found",
__FUNCTION__, __LINE__, Id);
return gcvSTATUS_NOT_FOUND;
}
}
gceSTATUS
_DestroyIntegerId(
IN gcsINTEGER_DB_PTR Database,
IN gctUINT32 Id
)
{
spin_lock(&Database->lock);
idr_remove(&Database->idr, Id);
spin_unlock(&Database->lock);
return gcvSTATUS_OK;
}
static inline gceSTATUS
_QueryProcessPageTable(
IN gctPOINTER Logical,
OUT gctPHYS_ADDR_T * Address
)
{
unsigned long logical = (unsigned long)Logical;
unsigned long offset = logical & ~PAGE_MASK;
if (is_vmalloc_addr(Logical))
{
/* vmalloc area. */
*Address = page_to_phys(vmalloc_to_page(Logical)) | offset;
return gcvSTATUS_OK;
}
else if (virt_addr_valid(logical))
{
/* Kernel logical address. */
*Address = virt_to_phys(Logical);
return gcvSTATUS_OK;
}
else
{
/* Try user VM area. */
struct vm_area_struct *vma;
spinlock_t *ptl;
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
if (!current->mm)
return gcvSTATUS_NOT_FOUND;
down_read(&current->mm->mmap_sem);
vma = find_vma(current->mm, logical);
up_read(&current->mm->mmap_sem);
/* To check if mapped to user. */
if (!vma)
return gcvSTATUS_NOT_FOUND;
pgd = pgd_offset(current->mm, logical);
if (pgd_none(*pgd) || pgd_bad(*pgd))
return gcvSTATUS_NOT_FOUND;
#if (defined(CONFIG_CPU_CSKYV2) || defined(CONFIG_X86)) \
&& LINUX_VERSION_CODE >= KERNEL_VERSION (4,12,0)
pud = pud_offset((p4d_t*)pgd, logical);
#elif (defined(CONFIG_CPU_CSKYV2)) \
&& LINUX_VERSION_CODE >= KERNEL_VERSION (4,11,0)
pud = pud_offset((p4d_t*)pgd, logical);
#else
pud = pud_offset(pgd, logical);
#endif
if (pud_none(*pud) || pud_bad(*pud))
return gcvSTATUS_NOT_FOUND;
pmd = pmd_offset(pud, logical);
if (pmd_none(*pmd) || pmd_bad(*pmd))
return gcvSTATUS_NOT_FOUND;
pte = pte_offset_map_lock(current->mm, pmd, logical, &ptl);
if (!pte)
{
spin_unlock(ptl);
return gcvSTATUS_NOT_FOUND;
}
if (!pte_present(*pte))
{
pte_unmap_unlock(pte, ptl);
return gcvSTATUS_NOT_FOUND;
}
*Address = (pte_pfn(*pte) << PAGE_SHIFT) | offset;
pte_unmap_unlock(pte, ptl);
return gcvSTATUS_OK;
}
}
static gceSTATUS
_ShrinkMemory(
IN gckOS Os
)
{
gcsPLATFORM * platform;
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Os=%p", Os);
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
platform = Os->device->platform;
if (platform && platform->ops->shrinkMemory)
{
status = platform->ops->shrinkMemory(platform);
}
else
{
status = gcvSTATUS_NOT_SUPPORTED;
}
gcmkFOOTER();
return status;
}
#if gcdDUMP_IN_KERNEL
#define DUMP_TO_KERNEL_DMESG 0
#define DUMP_TO_FILE 1
#define DUMP_IGNORE 2
static void set_dump_file(gckOS os, char * fname)
{
if (os->dumpFilp)
{
/* close exist opened file. */
printk("galcore: end dump to file: %s\n", os->dumpFileName);
filp_close(os->dumpFilp, NULL);
os->dumpFilp = NULL;
}
if (fname[0] == '\0' || !strcmp(fname, "[ignored]"))
{
/* empty or [ignored] means ignored. */
printk("galcore: dump ignored\n");
os->dumpTarget = DUMP_IGNORE;
strcpy(os->dumpFileName, "[ignored]");
}
else if (!strcmp(fname, "[dmesg]"))
{
/* [dmesg] means dump to kernel dmesg. */
printk("galcore: dump to kernel dmesg\n");
os->dumpTarget = DUMP_TO_KERNEL_DMESG;
strcpy(os->dumpFileName, "[dmesg]");
}
else if (fname[0] != '/')
{
/* invalid path, switch to kernel dmesg. */
printk(KERN_ERR "galcore: invalid path: %s\n", fname);
printk(KERN_ERR "galcore: must be absolute path start with '/'\n");
printk("galcore: dump to kernel dmesg\n");
os->dumpTarget = DUMP_TO_KERNEL_DMESG;
strcpy(os->dumpFileName, "[dmesg]");
}
else
{
/* try open file. */
os->dumpFilp = filp_open(fname, O_RDWR | O_CREAT, 0644);
if (IS_ERR(os->dumpFilp))
{
printk(KERN_ERR "galcore: failed to open file: %s\n", fname);
printk("galcore: dump to kernel dmesg\n");
os->dumpFilp = NULL;
os->dumpTarget = DUMP_TO_KERNEL_DMESG;
strcpy(os->dumpFileName, "[dmesg]");
}
else
{
printk("galcore: start dump to file: %s\n", fname);
os->dumpTarget = DUMP_TO_FILE;
strcpy(os->dumpFileName, fname);
}
}
}
static int dump_file_show(struct seq_file *m, void *unused)
{
gcsINFO_NODE *node = m->private;
gckOS os = node->device;
seq_printf(m, "%s\n", os->dumpFileName);
return 0;
}
static int dump_file_write(const char __user *buf, size_t count, void* data)
{
gcsINFO_NODE *node = data;
gckOS os = node->device;
char fname[256];
size_t len = min(count, sizeof(fname) - 1);
if (copy_from_user(fname, buf, len))
{
return -EFAULT;
}
/* Remove tailing space. */
while (len > 0 && (fname[len - 1] == '\n' || fname[len - 1] == ' '))
{
fname[len - 1] = '\0';
}
fname[len] = '\0';
mutex_lock(&os->dumpFilpMutex);
set_dump_file(os, fname);
mutex_unlock(&os->dumpFilpMutex);
return count;
}
static gcsINFO dumpDebugList[] =
{
{"dump_file", dump_file_show, dump_file_write},
};
static gceSTATUS
_DumpDebugfsInit(
IN gckOS Os
)
{
gceSTATUS status;
gckGALDEVICE device = Os->device;
gckDEBUGFS_DIR dir = &Os->dumpDebugfsDir;
gcmkONERROR(gckDEBUGFS_DIR_Init(dir, device->debugfsDir.root, "dump"));
gcmkONERROR(
gckDEBUGFS_DIR_CreateFiles(dir, dumpDebugList,
gcmCOUNTOF(dumpDebugList), Os));
OnError:
return status;
}
static void
_DumpDebugfsCleanup(
IN gckOS Os
)
{
gckDEBUGFS_DIR dir = &Os->dumpDebugfsDir;
if (dir->root)
{
gckDEBUGFS_DIR_RemoveFiles(dir, dumpDebugList, gcmCOUNTOF(dumpDebugList));
gckDEBUGFS_DIR_Deinit(dir);
}
}
#endif
/*******************************************************************************
**
** gckOS_Construct
**
** Construct a new gckOS object.
**
** INPUT:
**
** gctPOINTER Context
** Pointer to the gckGALDEVICE class.
**
** OUTPUT:
**
** gckOS * Os
** Pointer to a variable that will hold the pointer to the gckOS object.
*/
gceSTATUS
gckOS_Construct(
IN gctPOINTER Context,
OUT gckOS * Os
)
{
gckOS os = gcvNULL;
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Context=%p", Context);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(Os != gcvNULL);
/* Allocate the gckOS object. */
os = (gckOS)kmalloc(gcmSIZEOF(struct _gckOS), GFP_KERNEL | gcdNOWARN);
if (os == gcvNULL)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
/* Zero the memory. */
gckOS_ZeroMemory(os, gcmSIZEOF(struct _gckOS));
/* Initialize the gckOS object. */
os->object.type = gcvOBJ_OS;
/* Set device device. */
os->device = Context;
/* Set allocateCount to 0, gckOS_Allocate has not been used yet. */
atomic_set(&os->allocateCount, 0);
/* Initialize the memory lock. */
mutex_init(&os->mdlMutex);
INIT_LIST_HEAD(&os->mdlHead);
/* Get the kernel process ID. */
os->kernelProcessID = _GetProcessID();
/*
* Initialize the signal manager.
*/
/* Initialize spinlock. */
spin_lock_init(&os->signalLock);
/* Initialize signal id database lock. */
spin_lock_init(&os->signalDB.lock);
/* Initialize signal id database. */
idr_init(&os->signalDB.idr);
/* Create a workqueue for os timer. */
os->workqueue = create_singlethread_workqueue("galcore workqueue");
if (os->workqueue == gcvNULL)
{
/* Out of memory. */
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
os->paddingPage = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | gcdNOWARN);
if (os->paddingPage == gcvNULL)
{
/* Out of memory. */
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
else
{
SetPageReserved(os->paddingPage);
}
spin_lock_init(&os->registerAccessLock);
gckOS_ImportAllocators(os);
#if defined(CONFIG_IOMMU_SUPPORT)
if (0)
{
/* Only use IOMMU when internal MMU is not enabled. */
if (gcmIS_ERROR(gckIOMMU_Construct(os, &os->iommu)))
{
gcmkTRACE_ZONE(
gcvLEVEL_INFO, gcvZONE_OS,
"%s(%d): Fail to setup IOMMU",
__FUNCTION__, __LINE__
);
}
}
#endif
#if gcdDUMP_IN_KERNEL
mutex_init(&os->dumpFilpMutex);
/* Set default dump file. */
set_dump_file(os, gcdDUMP_FILE_IN_KERNEL);
/* Init debugfs for kernel dump feature. */
_DumpDebugfsInit(os);
#endif
/* Return pointer to the gckOS object. */
*Os = os;
OnError:
if (gcmIS_ERROR(status) && os)
{
if (os->workqueue != gcvNULL)
{
destroy_workqueue(os->workqueue);
}
kfree(os);
os = gcvNULL;
}
/* Return the error. */
gcmkFOOTER_ARG("*Os=%p", os);
return status;
}
/*******************************************************************************
**
** gckOS_Destroy
**
** Destroy an gckOS object.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object that needs to be destroyed.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_Destroy(
IN gckOS Os
)
{
gcmkHEADER_ARG("Os=%p", Os);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
if (Os->paddingPage != gcvNULL)
{
ClearPageReserved(Os->paddingPage);
__free_page(Os->paddingPage);
Os->paddingPage = gcvNULL;
}
/*
* Destroy the signal manager.
*/
/* Wait for all works done. */
flush_workqueue(Os->workqueue);
/* Destory work queue. */
destroy_workqueue(Os->workqueue);
gckOS_FreeAllocators(Os);
#ifdef CONFIG_IOMMU_SUPPORT
if (Os->iommu)
{
gckIOMMU_Destory(Os, Os->iommu);
}
#endif
/* Mark the gckOS object as unknown. */
Os->object.type = gcvOBJ_UNKNOWN;
#if gcdDUMP_IN_KERNEL
mutex_lock(&Os->dumpFilpMutex);
if (Os->dumpFilp)
{
filp_close(Os->dumpFilp, NULL);
Os->dumpFilp = NULL;
Os->dumpTarget = DUMP_IGNORE;
}
mutex_unlock(&Os->dumpFilpMutex);
/* Cleanup debugfs for kernel dump feature. */
_DumpDebugfsCleanup(Os);
#endif
/* Free the gckOS object. */
kfree(Os);
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
gceSTATUS
gckOS_CreateKernelMapping(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctSIZE_T Offset,
IN gctSIZE_T Bytes,
OUT gctPOINTER * Logical
)
{
gceSTATUS status = gcvSTATUS_OK;
PLINUX_MDL mdl = (PLINUX_MDL)Physical;
gckALLOCATOR allocator = mdl->allocator;
gcmkHEADER_ARG("Os=%p Physical=%p Offset=0x%zx Bytes=0x%zx",
Os, Physical, Offset, Bytes);
if (mdl->addr)
{
/* Already mapped whole memory. */
*Logical = (gctUINT8_PTR)mdl->addr + Offset;
}
else
{
gcmkONERROR(gcmALLOCATOR_MapKernel(allocator, mdl, Offset, Bytes, Logical));
}
OnError:
gcmkFOOTER_ARG("*Logical=%p", gcmOPT_POINTER(Logical));
return status;
}
gceSTATUS
gckOS_DestroyKernelMapping(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctPOINTER Logical
)
{
PLINUX_MDL mdl = (PLINUX_MDL)Physical;
gckALLOCATOR allocator = mdl->allocator;
gcmkHEADER_ARG("Os=%p Physical=%p Logical=%p", Os, Physical, Logical);
if (mdl->addr)
{
/* Nothing to do. */
}
else
{
gcmALLOCATOR_UnmapKernel(allocator, mdl, Logical);
}
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_Allocate
**
** Allocate memory.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctSIZE_T Bytes
** Number of bytes to allocate.
**
** OUTPUT:
**
** gctPOINTER * Memory
** Pointer to a variable that will hold the allocated memory location.
*/
gceSTATUS
gckOS_Allocate(
IN gckOS Os,
IN gctSIZE_T Bytes,
OUT gctPOINTER * Memory
)
{
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Os=%p Bytes=0x%zx", Os, Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Memory != gcvNULL);
gcmkONERROR(gckOS_AllocateMemory(Os, Bytes, Memory));
OnError:
/* Return the status. */
gcmkFOOTER_ARG("*Memory=%p", gcmOPT_POINTER(Memory));
return status;
}
/*******************************************************************************
**
** gckOS_Free
**
** Free allocated memory.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPOINTER Memory
** Pointer to memory allocation to free.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_Free(
IN gckOS Os,
IN gctPOINTER Memory
)
{
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Os=%p Memory=%p", Os, Memory);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Memory != gcvNULL);
gcmkONERROR(gckOS_FreeMemory(Os, Memory));
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckOS_AllocateMemory
**
** Allocate memory wrapper.
**
** INPUT:
**
** gctSIZE_T Bytes
** Number of bytes to allocate.
**
** OUTPUT:
**
** gctPOINTER * Memory
** Pointer to a variable that will hold the allocated memory location.
*/
gceSTATUS
gckOS_AllocateMemory(
IN gckOS Os,
IN gctSIZE_T Bytes,
OUT gctPOINTER * Memory
)
{
gctPOINTER memory = gcvNULL;
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Os=%p Bytes=0x%zx", Os, Bytes);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Memory != gcvNULL);
if (Bytes > PAGE_SIZE)
{
memory = (gctPOINTER) vmalloc(Bytes);
}
else
{
memory = (gctPOINTER) kmalloc(Bytes, GFP_KERNEL | gcdNOWARN);
}
if (memory == gcvNULL)
{
/* Out of memory. */
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
/* Increase count. */
atomic_inc(&Os->allocateCount);
/* Return pointer to the memory allocation. */
*Memory = memory;
OnError:
/* Return the status. */
gcmkFOOTER_ARG("*Memory=%p", gcmOPT_POINTER(Memory));
return status;
}
/*******************************************************************************
**
** gckOS_FreeMemory
**
** Free allocated memory wrapper.
**
** INPUT:
**
** gctPOINTER Memory
** Pointer to memory allocation to free.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_FreeMemory(
IN gckOS Os,
IN gctPOINTER Memory
)
{
gcmkHEADER_ARG("Os=%p Memory=%p", Os, Memory);
/* Verify the arguments. */
gcmkVERIFY_ARGUMENT(Memory != gcvNULL);
/* Free the memory from the OS pool. */
if (is_vmalloc_addr(Memory))
{
vfree(Memory);
}
else
{
kfree(Memory);
}
/* Decrease count. */
atomic_dec(&Os->allocateCount);
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_MapMemory
**
** Map physical memory into the current process.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR Physical
** Start of physical address memory.
**
** gctSIZE_T Bytes
** Number of bytes to map.
**
** OUTPUT:
**
** gctPOINTER * Memory
** Pointer to a variable that will hold the logical address of the
** mapped memory.
*/
gceSTATUS
gckOS_MapMemory(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctSIZE_T Bytes,
OUT gctPOINTER * Logical
)
{
gceSTATUS status = gcvSTATUS_OK;
PLINUX_MDL_MAP mdlMap;
PLINUX_MDL mdl = (PLINUX_MDL) Physical;
gckALLOCATOR allocator;
gctINT pid = _GetProcessID();
gcmkHEADER_ARG("Os=%p Physical=%p Bytes=0x%zx", Os, Physical, Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Physical != 0);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
mutex_lock(&mdl->mapsMutex);
mdlMap = FindMdlMap(mdl, pid);
if (mdlMap == gcvNULL)
{
mdlMap = _CreateMdlMap(mdl, pid);
if (mdlMap == gcvNULL)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
}
if (mdlMap->vmaAddr == gcvNULL)
{
allocator = mdl->allocator;
gcmkONERROR(gcmALLOCATOR_MapUser(allocator, mdl, mdlMap, gcvFALSE));
}
mutex_unlock(&mdl->mapsMutex);
*Logical = mdlMap->vmaAddr;
gcmkFOOTER_ARG("*Logical=%p", Logical);
return gcvSTATUS_OK;
OnError:
mutex_unlock(&mdl->mapsMutex);
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckOS_UnmapMemory
**
** Unmap physical memory out of the current process.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR Physical
** Start of physical address memory.
**
** gctSIZE_T Bytes
** Number of bytes to unmap.
**
** gctPOINTER Memory
** Pointer to a previously mapped memory region.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_UnmapMemory(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctSIZE_T Bytes,
IN gctPOINTER Logical
)
{
gcmkHEADER_ARG("Os=%p Physical=0%p Bytes=0x%zx Logical=%p",
Os, Physical, Bytes, Logical);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Physical != 0);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
gckOS_UnmapMemoryEx(Os, Physical, Bytes, Logical, _GetProcessID());
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_UnmapMemoryEx
**
** Unmap physical memory in the specified process.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR Physical
** Start of physical address memory.
**
** gctSIZE_T Bytes
** Number of bytes to unmap.
**
** gctPOINTER Memory
** Pointer to a previously mapped memory region.
**
** gctUINT32 PID
** Pid of the process that opened the device and mapped this memory.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_UnmapMemoryEx(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctSIZE_T Bytes,
IN gctPOINTER Logical,
IN gctUINT32 PID
)
{
PLINUX_MDL_MAP mdlMap;
PLINUX_MDL mdl = (PLINUX_MDL)Physical;
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Os=%p Physical=%p Bytes=0x%zx Logical=%p PID=%d",
Os, Physical, Bytes, Logical, PID);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Physical != 0);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
gcmkVERIFY_ARGUMENT(PID != 0);
if (Logical)
{
gckALLOCATOR allocator = mdl->allocator;
mutex_lock(&mdl->mapsMutex);
mdlMap = FindMdlMap(mdl, PID);
if (mdlMap == gcvNULL)
{
mutex_unlock(&mdl->mapsMutex);
gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
}
if (mdlMap->vmaAddr != gcvNULL)
{
BUG_ON(!allocator || !allocator->ops->UnmapUser);
gcmALLOCATOR_UnmapUser(allocator, mdl, mdlMap, mdl->bytes);
}
gcmkVERIFY_OK(_DestroyMdlMap(mdl, mdlMap));
mutex_unlock(&mdl->mapsMutex);
}
OnError:
gcmkFOOTER_NO();
return status;
}
/*******************************************************************************
**
** gckOS_AllocateNonPagedMemory
**
** Allocate a number of pages from non-paged memory.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctBOOL InUserSpace
** gcvTRUE if the pages need to be mapped into user space.
**
** gctUINT32 Flag
** Allocation attribute.
**
** gctSIZE_T * Bytes
** Pointer to a variable that holds the number of bytes to allocate.
**
** OUTPUT:
**
** gctSIZE_T * Bytes
** Pointer to a variable that hold the number of bytes allocated.
**
** gctPHYS_ADDR * Physical
** Pointer to a variable that will hold the physical address of the
** allocation.
**
** gctPOINTER * Logical
** Pointer to a variable that will hold the logical address of the
** allocation.
*/
gceSTATUS
gckOS_AllocateNonPagedMemory(
IN gckOS Os,
IN gctBOOL InUserSpace,
IN gctUINT32 Flag,
IN OUT gctSIZE_T * Bytes,
OUT gctPHYS_ADDR * Physical,
OUT gctPOINTER * Logical
)
{
gctSIZE_T bytes;
gctSIZE_T numPages;
PLINUX_MDL mdl = gcvNULL;
PLINUX_MDL_MAP mdlMap = gcvNULL;
gctPOINTER addr;
gceSTATUS status = gcvSTATUS_NOT_SUPPORTED;
gckALLOCATOR allocator;
gcmkHEADER_ARG("Os=%p InUserSpace=%d *Bytes=0x%zx",
Os, InUserSpace, gcmOPT_VALUE(Bytes));
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Bytes != gcvNULL);
gcmkVERIFY_ARGUMENT(*Bytes > 0);
gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
/* Align number of bytes to page size. */
bytes = gcmALIGN(*Bytes, PAGE_SIZE);
/* Get total number of pages.. */
numPages = GetPageCount(bytes, 0);
/* Allocate mdl structure */
mdl = _CreateMdl(Os);
if (mdl == gcvNULL)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
gcmkASSERT(Flag & gcvALLOC_FLAG_CONTIGUOUS);
/* Walk all allocators. */
list_for_each_entry(allocator, &Os->allocatorList, link)
{
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_OS,
"%s(%d) flag = %x allocator->capability = %x",
__FUNCTION__, __LINE__, Flag, allocator->capability);
#ifndef NO_DMA_COHERENT
/* Point to dma coherent allocator. */
if (!strcmp(allocator->name, "dma") ||
((Flag & allocator->capability) == Flag && numPages == 1))
{
status = gcmALLOCATOR_Alloc(allocator, mdl, numPages, Flag);
if (gcmIS_SUCCESS(status))
{
mdl->allocator = allocator;
break;
}
}
#else
if ((Flag & allocator->capability) == Flag)
{
status = gcmALLOCATOR_Alloc(allocator, mdl, numPages, Flag);
if (gcmIS_SUCCESS(status))
{
mdl->allocator = allocator;
break;
}
}
#endif
}
/* Check status. */
gcmkONERROR(status);
mdl->cacheable = Flag & gcvALLOC_FLAG_CACHEABLE;
mdl->bytes = bytes;
mdl->numPages = numPages;
mdl->contiguous = gcvTRUE;
gcmkONERROR(gcmALLOCATOR_MapKernel(allocator, mdl, 0, bytes, &addr));
if (!strcmp(allocator->name, "gfp"))
{
/* Trigger a page fault. */
memset(addr, 0, numPages * PAGE_SIZE);
}
mdl->addr = addr;
if (InUserSpace)
{
mdlMap = _CreateMdlMap(mdl, _GetProcessID());
if (mdlMap == gcvNULL)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
gcmkONERROR(gcmALLOCATOR_MapUser(allocator, mdl, mdlMap, gcvFALSE));
*Logical = mdlMap->vmaAddr;
}
else
{
*Logical = addr;
}
/*
* Add this to a global list.
* Will be used by get physical address
* and mapuser pointer functions.
*/
mutex_lock(&Os->mdlMutex);
list_add_tail(&mdl->link, &Os->mdlHead);
mutex_unlock(&Os->mdlMutex);
/* Return allocated memory. */
*Bytes = bytes;
*Physical = (gctPHYS_ADDR) mdl;
/* Success. */
status = gcvSTATUS_OK;
OnError:
if (gcmIS_ERROR(status))
{
if (mdlMap)
{
/* Free LINUX_MDL_MAP. */
gcmkVERIFY_OK(_DestroyMdlMap(mdl, mdlMap));
}
if (mdl)
{
/* Free LINUX_MDL. */
gcmkVERIFY_OK(_DestroyMdl(mdl));
}
}
/* Return the status. */
gcmkFOOTER_ARG("*Bytes=0x%zx *Physical=%p *Logical=%p",
gcmOPT_VALUE(Bytes), gcmOPT_POINTER(Physical), gcmOPT_POINTER(Logical));
return status;
}
/*******************************************************************************
**
** gckOS_FreeNonPagedMemory
**
** Free previously allocated and mapped pages from non-paged memory.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR Physical
** Physical address of the allocated memory.
**
** gctPOINTER Logical
** Logical address of the allocated memory.
**
** gctSIZE_T Bytes
** Number of bytes allocated.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS gckOS_FreeNonPagedMemory(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctPOINTER Logical,
IN gctSIZE_T Bytes
)
{
PLINUX_MDL mdl = (PLINUX_MDL)Physical;
gcmkHEADER_ARG("Os=%p Bytes=0x%zx Physical=%p Logical=%p",
Os, Bytes, Physical, Logical);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Physical != 0);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
gcmkVERIFY_OK(_DestroyMdl(mdl));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
static inline gckALLOCATOR
_FindAllocator(
gckOS Os,
gctUINT Flag
)
{
gckALLOCATOR allocator;
list_for_each_entry(allocator, &Os->allocatorList, link)
{
if ((allocator->capability & Flag) == Flag)
{
return allocator;
}
}
return gcvNULL;
}
gceSTATUS
gckOS_RequestReservedMemory(
gckOS Os,
gctPHYS_ADDR_T Start,
gctSIZE_T Size,
const char * Name,
gctBOOL Requested,
gctPOINTER * MemoryHandle
)
{
PLINUX_MDL mdl = gcvNULL;
gceSTATUS status;
gckALLOCATOR allocator;
gcsATTACH_DESC desc;
gcmkHEADER_ARG("start=0x%lx size=0x%lx name=%s", Start, Size, Name);
/* Round up to page size. */
Size = (Size + ~PAGE_MASK) & PAGE_MASK;
mdl = _CreateMdl(Os);
if (!mdl)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
desc.reservedMem.start = Start;
desc.reservedMem.size = Size;
desc.reservedMem.name = Name;
desc.reservedMem.requested = Requested;
allocator = _FindAllocator(Os, gcvALLOC_FLAG_LINUX_RESERVED_MEM);
if (!allocator)
{
gcmkPRINT("reserved-mem allocator not integrated!");
gcmkONERROR(gcvSTATUS_GENERIC_IO);
}
/* Call attach. */
gcmkONERROR(gcmALLOCATOR_Attach(allocator, &desc, mdl));
/* Assign alloator. */
mdl->allocator = allocator;
mdl->bytes = Size;
mdl->numPages = Size >> PAGE_SHIFT;
mdl->contiguous = gcvTRUE;
mdl->addr = gcvNULL;
mdl->dmaHandle = Start;
mdl->gid = 0;
/*
* Add this to a global list.
* Will be used by get physical address
* and mapuser pointer functions.
*/
mutex_lock(&Os->mdlMutex);
list_add_tail(&mdl->link, &Os->mdlHead);
mutex_unlock(&Os->mdlMutex);
*MemoryHandle = (void *)mdl;
gcmkFOOTER_NO();
return gcvSTATUS_OK;
OnError:
if (mdl)
{
gcmkVERIFY_OK(_DestroyMdl(mdl));
}
gcmkFOOTER();
return status;
}
void
gckOS_ReleaseReservedMemory(
gckOS Os,
gctPOINTER MemoryHandle
)
{
PLINUX_MDL mdl = (PLINUX_MDL)MemoryHandle;
if (mdl)
{
gcmkVERIFY_OK(_DestroyMdl(mdl));
}
}
/*******************************************************************************
**
** gckOS_ReadRegister
**
** Read data from a register.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctUINT32 Address
** Address of register.
**
** OUTPUT:
**
** gctUINT32 * Data
** Pointer to a variable that receives the data read from the register.
*/
gceSTATUS
gckOS_ReadRegister(
IN gckOS Os,
IN gctUINT32 Address,
OUT gctUINT32 * Data
)
{
return gckOS_ReadRegisterEx(Os, gcvCORE_MAJOR, Address, Data);
}
gceSTATUS
gckOS_ReadRegisterEx(
IN gckOS Os,
IN gceCORE Core,
IN gctUINT32 Address,
OUT gctUINT32 * Data
)
{
if (Address > Os->device->registerSizes[Core] - 1)
{
return gcvSTATUS_INVALID_ARGUMENT;
}
if (in_irq())
{
uint32_t data;
spin_lock(&Os->registerAccessLock);
if (unlikely(Os->clockStates[Core] == gcvFALSE))
{
spin_unlock(&Os->registerAccessLock);
/*
* Read register when external clock off:
* 1. In shared IRQ, read register may be called and that's not our irq.
*/
return gcvSTATUS_GENERIC_IO;
}
data = readl(Os->device->registerBases[Core]);
if (unlikely((data & 0x3) == 0x3))
{
spin_unlock(&Os->registerAccessLock);
/*
* Read register when internal clock off:
* a. In shared IRQ, read register may be called and that's not our irq.
* b. In some condition, when ISR handled normal FE/PE, PM thread could
* trun off internal clock before ISR read register of async FE. And
* then IRQ handler will call read register with internal clock off.
* So here we just skip for such case.
*/
return gcvSTATUS_GENERIC_IO;
}
*Data = readl((gctUINT8 *)Os->device->registerBases[Core] + Address);
spin_unlock(&Os->registerAccessLock);
}
else
{
unsigned long flags;
spin_lock_irqsave(&Os->registerAccessLock, flags);
if (unlikely(Os->clockStates[Core] == gcvFALSE))
{
spin_unlock_irqrestore(&Os->registerAccessLock, flags);
/*
* Read register when external clock off:
* 2. In non-irq context, register access should not be called,
* otherwise it's driver bug.
*/
printk(KERN_ERR "[galcore]: %s(%d) GPU[%d] external clock off",
__func__, __LINE__, Core);
gcmkBUG_ON(1);
return gcvSTATUS_GENERIC_IO;
}
*Data = readl((gctUINT8 *)Os->device->registerBases[Core] + Address);
spin_unlock_irqrestore(&Os->registerAccessLock, flags);
#if gcdDUMP_AHB_ACCESS
/* Dangerous to print in interrupt context, skip. */
gcmkPRINT("@[RD %d] %08x %08x", Core, Address, *Data);
#endif
}
/* Success. */
return gcvSTATUS_OK;
}
static gceSTATUS
_WriteRegisterEx(
IN gckOS Os,
IN gceCORE Core,
IN gctUINT32 Address,
IN gctUINT32 Data,
IN gctBOOL Dump
)
{
if (Address > Os->device->registerSizes[Core] - 1)
{
return gcvSTATUS_INVALID_ARGUMENT;
}
if (in_irq())
{
spin_lock(&Os->registerAccessLock);
if (unlikely(Os->clockStates[Core] == gcvFALSE))
{
spin_unlock(&Os->registerAccessLock);
printk(KERN_ERR "[galcore]: %s(%d) GPU[%d] external clock off",
__func__, __LINE__, Core);
/* Driver bug: register write when clock off. */
gcmkBUG_ON(1);
return gcvSTATUS_GENERIC_IO;
}
writel(Data, (gctUINT8 *)Os->device->registerBases[Core] + Address);
spin_unlock(&Os->registerAccessLock);
}
else
{
unsigned long flags;
if (Dump)
{
gcmkDUMP(Os, "@[register.write %u 0x%05X 0x%08X]", Core, Address, Data);
}
spin_lock_irqsave(&Os->registerAccessLock, flags);
if (unlikely(Os->clockStates[Core] == gcvFALSE))
{
spin_unlock_irqrestore(&Os->registerAccessLock, flags);
printk(KERN_ERR "[galcore]: %s(%d) GPU[%d] external clock off",
__func__, __LINE__, Core);
/* Driver bug: register write when clock off. */
gcmkBUG_ON(1);
return gcvSTATUS_GENERIC_IO;
}
writel(Data, (gctUINT8 *)Os->device->registerBases[Core] + Address);
spin_unlock_irqrestore(&Os->registerAccessLock, flags);
#if gcdDUMP_AHB_ACCESS
/* Dangerous to print in interrupt context, skip. */
gcmkPRINT("@[WR %d] %08x %08x", Core, Address, Data);
#endif
}
/* Success. */
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_WriteRegister
**
** Write data to a register.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctUINT32 Address
** Address of register.
**
** gctUINT32 Data
** Data for register.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_WriteRegister(
IN gckOS Os,
IN gctUINT32 Address,
IN gctUINT32 Data
)
{
return _WriteRegisterEx(Os, gcvCORE_MAJOR, Address, Data, gcvTRUE);
}
gceSTATUS
gckOS_WriteRegisterEx(
IN gckOS Os,
IN gceCORE Core,
IN gctUINT32 Address,
IN gctUINT32 Data
)
{
return _WriteRegisterEx(Os, Core, Address, Data, gcvTRUE);
}
gceSTATUS
gckOS_WriteRegisterEx_NoDump(
IN gckOS Os,
IN gceCORE Core,
IN gctUINT32 Address,
IN gctUINT32 Data
)
{
return _WriteRegisterEx(Os, Core, Address, Data, gcvFALSE);
}
/*******************************************************************************
**
** gckOS_GetPageSize
**
** Get the system's page size.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** OUTPUT:
**
** gctSIZE_T * PageSize
** Pointer to a variable that will receive the system's page size.
*/
gceSTATUS gckOS_GetPageSize(
IN gckOS Os,
OUT gctSIZE_T * PageSize
)
{
gcmkHEADER_ARG("Os=%p", Os);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(PageSize != gcvNULL);
/* Return the page size. */
*PageSize = (gctSIZE_T) PAGE_SIZE;
/* Success. */
gcmkFOOTER_ARG("*PageSize=0x%zx", *PageSize);
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** _GetPhysicalAddressProcess
**
** Get the physical system address of a corresponding virtual address for a
** given process.
**
** INPUT:
**
** gckOS Os
** Pointer to gckOS object.
**
** gctPOINTER Logical
** Logical address.
**
** gctUINT32 ProcessID
** Process ID.
**
** OUTPUT:
**
** gctUINT32 * Address
** Poinetr to a variable that receives the 32-bit physical adress.
*/
static gceSTATUS
_GetPhysicalAddressProcess(
IN gckOS Os,
IN gctPOINTER Logical,
IN gctUINT32 ProcessID,
OUT gctPHYS_ADDR_T * Address
)
{
PLINUX_MDL mdl;
gceSTATUS status = gcvSTATUS_INVALID_ADDRESS;
gcmkHEADER_ARG("Os=%p Logical=%p ProcessID=%d", Os, Logical, ProcessID);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Address != gcvNULL);
mutex_lock(&Os->mdlMutex);
if (Os->device->contiguousPhysical)
{
/* Try the contiguous memory pool. */
mdl = (PLINUX_MDL) Os->device->contiguousPhysical;
mutex_lock(&mdl->mapsMutex);
status = _ConvertLogical2Physical(Os, Logical, ProcessID, mdl, Address);
mutex_unlock(&mdl->mapsMutex);
}
if (gcmIS_ERROR(status))
{
/* Walk all MDLs. */
list_for_each_entry(mdl, &Os->mdlHead, link)
{
mutex_lock(&mdl->mapsMutex);
status = _ConvertLogical2Physical(Os, Logical, ProcessID, mdl, Address);
mutex_unlock(&mdl->mapsMutex);
if (gcmIS_SUCCESS(status))
{
break;
}
}
}
mutex_unlock(&Os->mdlMutex);
gcmkONERROR(status);
/* Success. */
OnError:
/* Return the status. */
gcmkFOOTER_ARG("*Address=%p", *Address);
return status;
}
/*******************************************************************************
**
** gckOS_GetPhysicalAddress
**
** Get the physical system address of a corresponding virtual address.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPOINTER Logical
** Logical address.
**
** OUTPUT:
**
** gctUINT32 * Address
** Poinetr to a variable that receives the 32-bit physical adress.
*/
gceSTATUS
gckOS_GetPhysicalAddress(
IN gckOS Os,
IN gctPOINTER Logical,
OUT gctPHYS_ADDR_T * Address
)
{
gceSTATUS status;
gctUINT32 processID;
gcmkHEADER_ARG("Os=%p Logical=%p", Os, Logical);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Address != gcvNULL);
/* Query page table of current process first. */
status = _QueryProcessPageTable(Logical, Address);
if (gcmIS_ERROR(status))
{
/* Get current process ID. */
processID = _GetProcessID();
/* Route through other function. */
gcmkONERROR(
_GetPhysicalAddressProcess(Os, Logical, processID, Address));
}
/* Success. */
gcmkFOOTER_ARG("*Address=0x%llx", *Address);
return gcvSTATUS_OK;
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
gceSTATUS
gckOS_GetPhysicalFromHandle(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctUINT32 Offset,
OUT gctPHYS_ADDR_T * PhysicalAddress
)
{
PLINUX_MDL mdl = (PLINUX_MDL)Physical;
gckALLOCATOR allocator = mdl->allocator;
return gcmALLOCATOR_Physical(allocator, mdl, Offset, PhysicalAddress);
}
/*******************************************************************************
**
** gckOS_UserLogicalToPhysical
**
** Get the physical system address of a corresponding user virtual address.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPOINTER Logical
** Logical address.
**
** OUTPUT:
**
** gctUINT32 * Address
** Pointer to a variable that receives the 32-bit physical address.
*/
gceSTATUS gckOS_UserLogicalToPhysical(
IN gckOS Os,
IN gctPOINTER Logical,
OUT gctPHYS_ADDR_T * Address
)
{
return gckOS_GetPhysicalAddress(Os, Logical, Address);
}
gceSTATUS
_ConvertLogical2Physical(
IN gckOS Os,
IN gctPOINTER Logical,
IN gctUINT32 ProcessID,
IN PLINUX_MDL Mdl,
OUT gctPHYS_ADDR_T * Physical
)
{
gckALLOCATOR allocator = Mdl->allocator;
gctUINT32 offset;
gceSTATUS status = gcvSTATUS_NOT_FOUND;
gctINT8_PTR vBase;
/* TASK_SIZE is userspace - kernelspace virtual memory split. */
if ((gctUINTPTR_T)Logical >= TASK_SIZE)
{
/* Kernel virtual address. */
vBase = Mdl->addr;
}
else
{
/* User virtual address. */
PLINUX_MDL_MAP map;
map = FindMdlMap(Mdl, (gctINT) ProcessID);
vBase = (map == gcvNULL) ? gcvNULL : (gctINT8_PTR) map->vmaAddr;
}
/* Is the given address within that range. */
if ((vBase != gcvNULL)
&& ((gctINT8_PTR) Logical >= vBase)
&& ((gctINT8_PTR) Logical < vBase + Mdl->bytes)
)
{
offset = (gctINT8_PTR) Logical - vBase;
gcmALLOCATOR_Physical(allocator, Mdl, offset, Physical);
status = gcvSTATUS_OK;
}
return status;
}
/*******************************************************************************
**
** gckOS_MapPhysical
**
** Map a physical address into kernel space.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR_T Physical
** Physical address of the memory to map.
**
** gctSIZE_T Bytes
** Number of bytes to map.
**
** OUTPUT:
**
** gctPOINTER * Logical
** Pointer to a variable that receives the base address of the mapped
** memory.
*/
gceSTATUS
gckOS_MapPhysical(
IN gckOS Os,
IN gctPHYS_ADDR_T Physical,
IN gctSIZE_T Bytes,
OUT gctPOINTER * Logical
)
{
gctPOINTER logical;
PLINUX_MDL mdl;
gctBOOL found = gcvFALSE;
dma_addr_t physical = Physical;
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Os=%p Physical=0x%llx Bytes=0x%zx", Os, Physical, Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Bytes > 0);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
mutex_lock(&Os->mdlMutex);
/* Go through our mapping to see if we know this physical address already. */
list_for_each_entry(mdl, &Os->mdlHead, link)
{
if (mdl->dmaHandle != 0)
{
if ((physical >= mdl->dmaHandle)
&& (physical < mdl->dmaHandle + mdl->bytes)
&& (mdl->addr != 0)
)
{
*Logical = mdl->addr + (physical - mdl->dmaHandle);
found = gcvTRUE;
break;
}
}
}
mutex_unlock(&Os->mdlMutex);
if (!found)
{
unsigned long pfn = physical >> PAGE_SHIFT;
if (pfn_valid(pfn))
{
gctUINT32 offset = physical & ~PAGE_MASK;
struct page ** pages;
struct page * page;
gctSIZE_T numPages;
gctSIZE_T i;
pgprot_t pgprot;
numPages = GetPageCount(PAGE_ALIGN(offset + Bytes), 0);
pages = kmalloc(sizeof(struct page *) * numPages, GFP_KERNEL | gcdNOWARN);
if (!pages)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
page = pfn_to_page(pfn);
for (i = 0; i < numPages; i++)
{
pages[i] = nth_page(page, i);
}
#if gcdENABLE_BUFFERABLE_VIDEO_MEMORY
pgprot = pgprot_writecombine(PAGE_KERNEL);
#else
pgprot = pgprot_noncached(PAGE_KERNEL);
#endif
logical = vmap(pages, numPages, 0, pgprot);
kfree(pages);
if (logical == gcvNULL)
{
gcmkTRACE_ZONE(
gcvLEVEL_INFO, gcvZONE_OS,
"%s(%d): Failed to vmap",
__FUNCTION__, __LINE__
);
/* Out of resources. */
gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
}
logical += offset;
}
else
{
/* Map memory as cached memory. */
request_mem_region(physical, Bytes, "MapRegion");
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,5,0)
logical = (gctPOINTER) ioremap(physical, Bytes);
#else
logical = (gctPOINTER) ioremap_nocache(physical, Bytes);
#endif
if (logical == gcvNULL)
{
gcmkTRACE_ZONE(
gcvLEVEL_INFO, gcvZONE_OS,
"%s(%d): Failed to ioremap",
__FUNCTION__, __LINE__
);
/* Out of resources. */
gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
}
}
/* Return pointer to mapped memory. */
*Logical = logical;
}
OnError:
/* Success. */
gcmkFOOTER_ARG("*Logical=%p", *Logical);
return status;
}
/*******************************************************************************
**
** gckOS_UnmapPhysical
**
** Unmap a previously mapped memory region from kernel memory.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPOINTER Logical
** Pointer to the base address of the memory to unmap.
**
** gctSIZE_T Bytes
** Number of bytes to unmap.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_UnmapPhysical(
IN gckOS Os,
IN gctPOINTER Logical,
IN gctSIZE_T Bytes
)
{
PLINUX_MDL mdl;
gctBOOL found = gcvFALSE;
gcmkHEADER_ARG("Os=%p Logical=%p Bytes=0x%zx", Os, Logical, Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
gcmkVERIFY_ARGUMENT(Bytes > 0);
mutex_lock(&Os->mdlMutex);
list_for_each_entry(mdl, &Os->mdlHead, link)
{
if (mdl->addr != gcvNULL)
{
if ((Logical >= (gctPOINTER)mdl->addr) &&
(Logical < (gctPOINTER)((gctSTRING)mdl->addr + mdl->bytes)))
{
found = gcvTRUE;
break;
}
}
}
mutex_unlock(&Os->mdlMutex);
if (!found)
{
/* Unmap the memory. */
vunmap((void *)((unsigned long)Logical & PAGE_MASK));
}
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_DeleteMutex
**
** Delete a mutex.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPOINTER Mutex
** Pointer to the mute to be deleted.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_DeleteMutex(
IN gckOS Os,
IN gctPOINTER Mutex
)
{
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Os=%p Mutex=%p", Os, Mutex);
/* Validate the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Mutex != gcvNULL);
/* Destroy the mutex. */
mutex_destroy((struct mutex *)Mutex);
/* Free the mutex structure. */
gcmkONERROR(gckOS_Free(Os, Mutex));
OnError:
/* Return status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckOS_AcquireMutex
**
** Acquire a mutex.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPOINTER Mutex
** Pointer to the mutex to be acquired.
**
** gctUINT32 Timeout
** Timeout value specified in milliseconds.
** Specify the value of gcvINFINITE to keep the thread suspended
** until the mutex has been acquired.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_AcquireMutex(
IN gckOS Os,
IN gctPOINTER Mutex,
IN gctUINT32 Timeout
)
{
gceSTATUS status = gcvSTATUS_TIMEOUT;
gcmkHEADER_ARG("Os=%p Mutex=%p Timeout=%u", Os, Mutex, Timeout);
/* Validate the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Mutex != gcvNULL);
if (Timeout == gcvINFINITE)
{
/* Lock the mutex. */
mutex_lock(Mutex);
/* Success. */
status = gcvSTATUS_OK;
}
else
{
for (;;)
{
/* Try to acquire the mutex. */
if (mutex_trylock(Mutex))
{
/* Success. */
status = gcvSTATUS_OK;
break;
}
if (Timeout-- == 0)
{
break;
}
/* Wait for 1 millisecond. */
gcmkVERIFY_OK(gckOS_Delay(Os, 1));
}
}
/* Timeout. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckOS_ReleaseMutex
**
** Release an acquired mutex.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPOINTER Mutex
** Pointer to the mutex to be released.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_ReleaseMutex(
IN gckOS Os,
IN gctPOINTER Mutex
)
{
gcmkHEADER_ARG("Os=%p Mutex=%p", Os, Mutex);
/* Validate the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Mutex != gcvNULL);
/* Release the mutex. */
mutex_unlock(Mutex);
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_AtomicExchange
**
** Atomically exchange a pair of 32-bit values.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** IN OUT gctINT32_PTR Target
** Pointer to the 32-bit value to exchange.
**
** IN gctINT32 NewValue
** Specifies a new value for the 32-bit value pointed to by Target.
**
** OUT gctINT32_PTR OldValue
** The old value of the 32-bit value pointed to by Target.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_AtomicExchange(
IN gckOS Os,
IN OUT gctUINT32_PTR Target,
IN gctUINT32 NewValue,
OUT gctUINT32_PTR OldValue
)
{
/* Exchange the pair of 32-bit values. */
*OldValue = (gctUINT32) atomic_xchg((atomic_t *) Target, (int) NewValue);
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_AtomicExchangePtr
**
** Atomically exchange a pair of pointers.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** IN OUT gctPOINTER * Target
** Pointer to the 32-bit value to exchange.
**
** IN gctPOINTER NewValue
** Specifies a new value for the pointer pointed to by Target.
**
** OUT gctPOINTER * OldValue
** The old value of the pointer pointed to by Target.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_AtomicExchangePtr(
IN gckOS Os,
IN OUT gctPOINTER * Target,
IN gctPOINTER NewValue,
OUT gctPOINTER * OldValue
)
{
/* Exchange the pair of pointers. */
*OldValue = (gctPOINTER)(gctUINTPTR_T) atomic_xchg((atomic_t *) Target, (int)(gctUINTPTR_T) NewValue);
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_AtomicSetMask
**
** Atomically set mask to Atom
**
** INPUT:
** IN OUT gctPOINTER Atom
** Pointer to the atom to set.
**
** IN gctUINT32 Mask
** Mask to set.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_AtomSetMask(
IN gctPOINTER Atom,
IN gctUINT32 Mask
)
{
gctUINT32 oval, nval;
do
{
oval = atomic_read((atomic_t *) Atom);
nval = oval | Mask;
}
while (atomic_cmpxchg((atomic_t *) Atom, oval, nval) != oval);
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_AtomClearMask
**
** Atomically clear mask from Atom
**
** INPUT:
** IN OUT gctPOINTER Atom
** Pointer to the atom to clear.
**
** IN gctUINT32 Mask
** Mask to clear.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_AtomClearMask(
IN gctPOINTER Atom,
IN gctUINT32 Mask
)
{
gctUINT32 oval, nval;
do
{
oval = atomic_read((atomic_t *) Atom);
nval = oval & ~Mask;
}
while (atomic_cmpxchg((atomic_t *) Atom, oval, nval) != oval);
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_AtomConstruct
**
** Create an atom.
**
** INPUT:
**
** gckOS Os
** Pointer to a gckOS object.
**
** OUTPUT:
**
** gctPOINTER * Atom
** Pointer to a variable receiving the constructed atom.
*/
gceSTATUS
gckOS_AtomConstruct(
IN gckOS Os,
OUT gctPOINTER * Atom
)
{
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Os=%p", Os);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Atom != gcvNULL);
/* Allocate the atom. */
gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(atomic_t), Atom));
/* Initialize the atom. */
atomic_set((atomic_t *) *Atom, 0);
OnError:
/* Return the status. */
gcmkFOOTER_ARG("*Atom=%p", *Atom);
return status;
}
/*******************************************************************************
**
** gckOS_AtomDestroy
**
** Destroy an atom.
**
** INPUT:
**
** gckOS Os
** Pointer to a gckOS object.
**
** gctPOINTER Atom
** Pointer to the atom to destroy.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_AtomDestroy(
IN gckOS Os,
OUT gctPOINTER Atom
)
{
gceSTATUS status = gcvSTATUS_OK;
gcmkHEADER_ARG("Os=%p Atom=%p", Os, Atom);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Atom != gcvNULL);
/* Free the atom. */
gcmkONERROR(gcmkOS_SAFE_FREE(Os, Atom));
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckOS_AtomGet
**
** Get the 32-bit value protected by an atom.
**
** INPUT:
**
** gckOS Os
** Pointer to a gckOS object.
**
** gctPOINTER Atom
** Pointer to the atom.
**
** OUTPUT:
**
** gctINT32_PTR Value
** Pointer to a variable the receives the value of the atom.
*/
gceSTATUS
gckOS_AtomGet(
IN gckOS Os,
IN gctPOINTER Atom,
OUT gctINT32_PTR Value
)
{
/* Return the current value of atom. */
*Value = atomic_read((atomic_t *) Atom);
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_AtomSet
**
** Set the 32-bit value protected by an atom.
**
** INPUT:
**
** gckOS Os
** Pointer to a gckOS object.
**
** gctPOINTER Atom
** Pointer to the atom.
**
** gctINT32 Value
** The value of the atom.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_AtomSet(
IN gckOS Os,
IN gctPOINTER Atom,
IN gctINT32 Value
)
{
/* Set the current value of atom. */
atomic_set((atomic_t *) Atom, Value);
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_AtomIncrement
**
** Atomically increment the 32-bit integer value inside an atom.
**
** INPUT:
**
** gckOS Os
** Pointer to a gckOS object.
**
** gctPOINTER Atom
** Pointer to the atom.
**
** OUTPUT:
**
** gctINT32_PTR Value
** Pointer to a variable that receives the original value of the atom.
*/
gceSTATUS
gckOS_AtomIncrement(
IN gckOS Os,
IN gctPOINTER Atom,
OUT gctINT32_PTR Value
)
{
*Value = atomic_inc_return((atomic_t *) Atom) - 1;
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_AtomDecrement
**
** Atomically decrement the 32-bit integer value inside an atom.
**
** INPUT:
**
** gckOS Os
** Pointer to a gckOS object.
**
** gctPOINTER Atom
** Pointer to the atom.
**
** OUTPUT:
**
** gctINT32_PTR Value
** Pointer to a variable that receives the original value of the atom.
*/
gceSTATUS
gckOS_AtomDecrement(
IN gckOS Os,
IN gctPOINTER Atom,
OUT gctINT32_PTR Value
)
{
/* Decrement the atom. */
*Value = atomic_dec_return((atomic_t *) Atom) + 1;
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_Delay
**
** Delay execution of the current thread for a number of milliseconds.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctUINT32 Delay
** Delay to sleep, specified in milliseconds.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_Delay(
IN gckOS Os,
IN gctUINT32 Delay
)
{
gcmkHEADER_ARG("Os=%p Delay=%u", Os, Delay);
if (Delay > 0)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
ktime_t delay = ktime_set((Delay / MSEC_PER_SEC), (Delay % MSEC_PER_SEC) * NSEC_PER_MSEC);
__set_current_state(TASK_UNINTERRUPTIBLE);
schedule_hrtimeout(&delay, HRTIMER_MODE_REL);
#else
msleep(Delay);
#endif
}
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_GetTicks
**
** Get the number of milliseconds since the system started.
**
** INPUT:
**
** OUTPUT:
**
** gctUINT32_PTR Time
** Pointer to a variable to get time.
**
*/
gceSTATUS
gckOS_GetTicks(
OUT gctUINT32_PTR Time
)
{
gcmkHEADER();
*Time = jiffies_to_msecs(jiffies);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_TicksAfter
**
** Compare time values got from gckOS_GetTicks.
**
** INPUT:
** gctUINT32 Time1
** First time value to be compared.
**
** gctUINT32 Time2
** Second time value to be compared.
**
** OUTPUT:
**
** gctBOOL_PTR IsAfter
** Pointer to a variable to result.
**
*/
gceSTATUS
gckOS_TicksAfter(
IN gctUINT32 Time1,
IN gctUINT32 Time2,
OUT gctBOOL_PTR IsAfter
)
{
gcmkHEADER();
*IsAfter = time_after((unsigned long)Time1, (unsigned long)Time2);
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_GetTime
**
** Get the number of microseconds since the system started.
**
** INPUT:
**
** OUTPUT:
**
** gctUINT64_PTR Time
** Pointer to a variable to get time.
**
*/
gceSTATUS
gckOS_GetTime(
OUT gctUINT64_PTR Time
)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
struct timespec64 tv;
gcmkHEADER();
/* Return the time of day in microseconds. */
ktime_get_real_ts64(&tv);
*Time = (tv.tv_sec * 1000000ULL) + (tv.tv_nsec / 1000);
#else
struct timeval tv;
gcmkHEADER();
/* Return the time of day in microseconds. */
do_gettimeofday(&tv);
*Time = (tv.tv_sec * 1000000ULL) + tv.tv_usec;
#endif
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_MemoryBarrier
**
** Make sure the CPU has executed everything up to this point and the data got
** written to the specified pointer.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPOINTER Address
** Address of memory that needs to be barriered.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_MemoryBarrier(
IN gckOS Os,
IN gctPOINTER Address
)
{
_MemoryBarrier();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_AllocatePagedMemory
**
** Allocate memory from the paged pool.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctUINT32 Flag
** Allocation attribute.
**
** gctSIZE_T * Bytes
** Number of bytes to allocate.
**
** OUTPUT:
**
** gctSIZE_T * Bytes
** Return number of bytes actually allocated.
**
** gctUINT32 * Gid
** Save the global ID for the piece of allocated memory.
**
** gctPHYS_ADDR * Physical
** Pointer to a variable that receives the physical address of the
** memory allocation.
*/
gceSTATUS
gckOS_AllocatePagedMemory(
IN gckOS Os,
IN gctUINT32 Flag,
IN OUT gctSIZE_T * Bytes,
OUT gctUINT32 * Gid,
OUT gctPHYS_ADDR * Physical
)
{
gctSIZE_T numPages;
PLINUX_MDL mdl = gcvNULL;
gctSIZE_T bytes;
gceSTATUS status = gcvSTATUS_NOT_SUPPORTED;
gckALLOCATOR allocator;
gctBOOL zoneDMA32 = gcvFALSE;
gcmkHEADER_ARG("Os=%p Flag=%x *Bytes=0x%zx", Os, Flag, *Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(*Bytes > 0);
gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
bytes = gcmALIGN(*Bytes, PAGE_SIZE);
numPages = GetPageCount(bytes, 0);
mdl = _CreateMdl(Os);
if (mdl == gcvNULL)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
#if defined(CONFIG_ZONE_DMA32) || defined(CONFIG_ZONE_DMA)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,37)
zoneDMA32 = gcvTRUE;
#endif
#endif
if ((Flag & gcvALLOC_FLAG_4GB_ADDR) && !zoneDMA32)
{
Flag &= ~gcvALLOC_FLAG_4GB_ADDR;
}
/* Walk all allocators. */
list_for_each_entry(allocator, &Os->allocatorList, link)
{
gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_OS,
"%s(%d) flag = %x allocator->capability = %x",
__FUNCTION__, __LINE__, Flag, allocator->capability);
if ((Flag & allocator->capability) != Flag)
{
continue;
}
status = gcmALLOCATOR_Alloc(allocator, mdl, numPages, Flag);
if (gcmIS_SUCCESS(status))
{
mdl->allocator = allocator;
break;
}
}
/* Check status. */
gcmkONERROR(status);
mdl->dmaHandle = 0;
mdl->addr = 0;
mdl->bytes = bytes;
mdl->numPages = numPages;
mdl->contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS;
mdl->cacheable = Flag & gcvALLOC_FLAG_CACHEABLE;
/*
* Add this to a global list.
* Will be used by get physical address
* and mapuser pointer functions.
*/
mutex_lock(&Os->mdlMutex);
list_add_tail(&mdl->link, &Os->mdlHead);
mutex_unlock(&Os->mdlMutex);
/* Return allocated bytes. */
*Bytes = bytes;
if (Gid != gcvNULL)
{
*Gid = mdl->gid;
}
/* Return physical address. */
*Physical = (gctPHYS_ADDR) mdl;
/* Success. */
status = gcvSTATUS_OK;
OnError:
if (gcmIS_ERROR(status) && mdl)
{
/* Free the memory. */
_DestroyMdl(mdl);
}
/* Return the status. */
gcmkFOOTER_ARG("*Physical=%p", *Physical);
return status;
}
/*******************************************************************************
**
** gckOS_FreePagedMemory
**
** Free memory allocated from the paged pool.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR Physical
** Physical address of the allocation.
**
** gctSIZE_T Bytes
** Number of bytes of the allocation.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_FreePagedMemory(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctSIZE_T Bytes
)
{
PLINUX_MDL mdl = (PLINUX_MDL)Physical;
gcmkHEADER_ARG("Os=%p Physical=%p Bytes=0x%zx", Os, Physical, Bytes);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
gcmkVERIFY_ARGUMENT(Bytes > 0);
/* Free the structure... */
gcmkVERIFY_OK(_DestroyMdl(mdl));
/* Success. */
gcmkFOOTER_NO();
return gcvSTATUS_OK;
}
/*******************************************************************************
**
** gckOS_LockPages
**
** Lock memory allocated from the paged pool.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR Physical
** Physical address of the allocation.
**
** gctSIZE_T Bytes
** Number of bytes of the allocation.
**
** gctBOOL Cacheable
** Cache mode of mapping.
**
** OUTPUT:
**
** gctPOINTER * Logical
** Pointer to a variable that receives the address of the mapped
** memory.
*/
gceSTATUS
gckOS_LockPages(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctSIZE_T Bytes,
IN gctBOOL Cacheable,
OUT gctPOINTER * Logical
)
{
gceSTATUS status = gcvSTATUS_OK;
PLINUX_MDL mdl;
PLINUX_MDL_MAP mdlMap;
gckALLOCATOR allocator;
gcmkHEADER_ARG("Os=%p Physical=%p Bytes=0x%zx", Os, Physical, Logical);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
mdl = (PLINUX_MDL)Physical;
allocator = mdl->allocator;
mutex_lock(&mdl->mapsMutex);
mdlMap = FindMdlMap(mdl, _GetProcessID());
if (mdlMap == gcvNULL)
{
mdlMap = _CreateMdlMap(mdl, _GetProcessID());
if (mdlMap == gcvNULL)
{
gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
}
}
if (mdlMap->vmaAddr == gcvNULL)
{
gcmkONERROR(gcmALLOCATOR_MapUser(allocator, mdl, mdlMap, Cacheable));
}
mdlMap->count++;
/* Convert pointer to MDL. */
*Logical = mdlMap->vmaAddr;
OnError:
mutex_unlock(&mdl->mapsMutex);
/* Success. */
gcmkFOOTER_ARG("*Logical=%p", *Logical);
return status;
}
/* PageCount is GPU page count. */
gceSTATUS
gckOS_MapPagesEx(
IN gckOS Os,
IN gceCORE Core,
IN gctPHYS_ADDR Physical,
IN gctSIZE_T PageCount,
IN gctUINT32 Address,
IN gctPOINTER PageTable,
IN gctBOOL Writable,
IN gceVIDMEM_TYPE Type
)
{
gceSTATUS status = gcvSTATUS_OK;
PLINUX_MDL mdl;
gctUINT32* table;
gctUINT32 offset = 0;
gctUINT32 bytes = PageCount * 4;
gckALLOCATOR allocator;
gctUINT32 policyID = 0;
gctUINT32 axiConfig = 0;
gcsPLATFORM * platform = Os->device->platform;
gcmkHEADER_ARG("Os=%p Core=%d Physical=%p PageCount=0x%zx Address=0x%x PageTable=%p",
Os, Core, Physical, PageCount, Address, PageTable);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
gcmkVERIFY_ARGUMENT(PageCount > 0);
gcmkVERIFY_ARGUMENT(PageTable != gcvNULL);
/* Convert pointer to MDL. */
mdl = (PLINUX_MDL)Physical;
allocator = mdl->allocator;
gcmkASSERT(allocator != gcvNULL);
gcmkTRACE_ZONE(
gcvLEVEL_INFO, gcvZONE_OS,
"%s(%d): Physical->0x%X PageCount->0x%X",
__FUNCTION__, __LINE__,
(gctUINT32)(gctUINTPTR_T)Physical,
(gctUINT32)(gctUINTPTR_T)PageCount
);
table = (gctUINT32 *)PageTable;
if (platform && platform->ops->getPolicyID)
{
platform->ops->getPolicyID(platform, Type, &policyID, &axiConfig);
gcmkBUG_ON(policyID > 0x1F);
/* ID[3:0] is used in STLB. */
policyID &= 0xF;
}
/* Get all the physical addresses and store them in the page table. */
PageCount = PageCount / (PAGE_SIZE / 4096);
/* Try to get the user pages so DMA can happen. */
while (PageCount-- > 0)
{
gctUINT i;
gctPHYS_ADDR_T phys = ~0U;
gcmALLOCATOR_Physical(allocator, mdl, offset, &phys);
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Os, phys, &phys));
if (policyID)
{
/* AxUSER must not used for address currently. */
gcmkBUG_ON((phys >> 32) & 0xF);
/* Merge policyID to AxUSER[7:4].*/
phys |= ((gctPHYS_ADDR_T)policyID << 36);
}
#ifdef CONFIG_IOMMU_SUPPORT
if (Os->iommu)
{
/* remove LSB. */
phys &= PAGE_MASK;
gcmkTRACE_ZONE(
gcvLEVEL_INFO, gcvZONE_OS,
"%s(%d): Setup mapping in IOMMU %x => %x",
__FUNCTION__, __LINE__,
Address + offset, phys
);
/* When use IOMMU, GPU use system PAGE_SIZE. */
gcmkONERROR(gckIOMMU_Map(
Os->iommu, Address + offset, phys, PAGE_SIZE));
}
else
#endif
{
/* remove LSB. */
phys &= ~(4096ull - 1);
{
for (i = 0; i < (PAGE_SIZE / 4096); i++)
{
gcmkONERROR(
gckMMU_SetPage(Os->device->kernels[Core]->mmu,
phys + (i * 4096),
gcvPAGE_TYPE_4K,
Writable,
table++));
}
}
}
offset += PAGE_SIZE;
}
{
gckMMU mmu = Os->device->kernels[Core]->mmu;
gcsADDRESS_AREA * area = &mmu->dynamicArea4K;
offset = (gctUINT8_PTR)PageTable - (gctUINT8_PTR)area->stlbLogical;
/* must be in dynamic area. */
gcmkASSERT(offset < area->stlbSize);
gcmkVERIFY_OK(gckVIDMEM_NODE_CleanCache(
Os->device->kernels[Core],
area->stlbVideoMem,
offset,
PageTable,
bytes
));
if (mmu->mtlbVideoMem)
{
/* Flush MTLB table. */
gcmkVERIFY_OK(gckVIDMEM_NODE_CleanCache(
Os->device->kernels[Core],
mmu->mtlbVideoMem,
offset,
mmu->mtlbLogical,
mmu->mtlbSize
));
}
}
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/* PageCount is GPU page count. */
gceSTATUS
gckOS_UnmapPages(
IN gckOS Os,
IN gctSIZE_T PageCount,
IN gctUINT32 Address
)
{
#ifdef CONFIG_IOMMU_SUPPORT
if (Os->iommu)
{
gcmkVERIFY_OK(gckIOMMU_Unmap(
Os->iommu, Address, PageCount * 4096));
}
#endif
return gcvSTATUS_OK;
}
/* Map 1M size GPU page */
gceSTATUS
gckOS_Map1MPages(
IN gckOS Os,
IN gceCORE Core,
IN gctPHYS_ADDR Physical,
IN gctSIZE_T PageCount,
IN gctUINT32 Address,
IN gctPOINTER PageTable,
IN gctBOOL Writable,
IN gceVIDMEM_TYPE Type
)
{
gceSTATUS status = gcvSTATUS_OK;
PLINUX_MDL mdl;
gctUINT32* table;
gctUINT32 offset = 0;
gctSIZE_T bytes = PageCount * 4;
gckALLOCATOR allocator;
gctUINT32 policyID = 0;
gctUINT32 axiConfig = 0;
gcsPLATFORM * platform = Os->device->platform;
gcmkHEADER_ARG("Os=%p Core=%d Physical=%p PageCount=0x%zx Address=0x%x PageTable=%p",
Os, Core, Physical, PageCount, Address, PageTable);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
gcmkVERIFY_ARGUMENT(PageCount > 0);
gcmkVERIFY_ARGUMENT(PageTable != gcvNULL);
/* Convert pointer to MDL. */
mdl = (PLINUX_MDL)Physical;
allocator = mdl->allocator;
gcmkASSERT(allocator != gcvNULL);
gcmkTRACE_ZONE(
gcvLEVEL_INFO, gcvZONE_OS,
"%s(%d): Physical->0x%X PageCount->0x%X",
__FUNCTION__, __LINE__,
(gctUINT32)(gctUINTPTR_T)Physical,
(gctUINT32)(gctUINTPTR_T)PageCount
);
table = (gctUINT32 *)PageTable;
if (platform && platform->ops->getPolicyID)
{
platform->ops->getPolicyID(platform, Type, &policyID, &axiConfig);
gcmkBUG_ON(policyID > 0x1F);
/* ID[3:0] is used in STLB. */
policyID &= 0xF;
}
while (PageCount-- > 0)
{
gctPHYS_ADDR_T phys = ~0U;
gcmALLOCATOR_Physical(allocator, mdl, offset, &phys);
gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(Os, phys, &phys));
if (policyID)
{
/* AxUSER must not used for address currently. */
gcmkBUG_ON((phys >> 32) & 0xF);
/* Merge policyID to AxUSER[7:4].*/
phys |= ((gctPHYS_ADDR_T)policyID << 36);
}
/* Get the start physical of 1M page. */
phys &= ~((1 << 20) - 1);
gcmkONERROR(
gckMMU_SetPage(Os->device->kernels[Core]->mmu,
phys,
gcvPAGE_TYPE_1M,
Writable,
table++));
offset += gcd1M_PAGE_SIZE;
}
/* Flush the page table cache. */
{
gckMMU mmu = Os->device->kernels[Core]->mmu;
gcsADDRESS_AREA * area = &mmu->dynamicArea1M;
offset = (gctUINT8_PTR)PageTable - (gctUINT8_PTR)area->stlbLogical;
/* must be in dynamic area. */
gcmkASSERT(offset < area->stlbSize);
gcmkVERIFY_OK(gckVIDMEM_NODE_CleanCache(
Os->device->kernels[Core],
area->stlbVideoMem,
offset,
PageTable,
bytes
));
if (mmu->mtlbVideoMem)
{
/* Flush MTLB table. */
gcmkVERIFY_OK(gckVIDMEM_NODE_CleanCache(
Os->device->kernels[Core],
mmu->mtlbVideoMem,
offset,
mmu->mtlbLogical,
mmu->mtlbSize
));
}
}
OnError:
/* Return the status. */
gcmkFOOTER();
return status;
}
/*******************************************************************************
**
** gckOS_UnlockPages
**
** Unlock memory allocated from the paged pool.
**
** INPUT:
**
** gckOS Os
** Pointer to an gckOS object.
**
** gctPHYS_ADDR Physical
** Physical address of the allocation.
**
** gctSIZE_T Bytes
** Number of bytes of the allocation.
**
** gctPOINTER Logical
** Address of the mapped memory.
**
** OUTPUT:
**
** Nothing.
*/
gceSTATUS
gckOS_UnlockPages(
IN gckOS Os,
IN gctPHYS_ADDR Physical,
IN gctSIZE_T Bytes,
IN gctPOINTER Logical
)
{
PLINUX_MDL_MAP mdlMap;
PLINUX_MDL mdl = (PLINUX_MDL)Physical;
gckALLOCATOR allocator = mdl->allocator;
gctINT pid = _GetProcessID();
gcmkHEADER_ARG("Os=%p Physical=%p Bytes=0x%zx Logical=%p",
Os, Physical, Bytes, Logical);
/* Verify the arguments. */
gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
mutex_lock(&mdl->mapsMutex);
list_for_each_entry(mdlMap, &mdl->mapsHead, link)
{
if ((mdlMap->vmaAddr != gcvNULL) && (mdlMap->pid == pid))
{
if (--mdlMap->count == 0)
{
gcmALLOCATOR_UnmapUser(