/*************************************************************************/ /*!
@File
@Title          Device Memory Management internal utility functions
@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
@Description    Utility functions used internally by device memory management
                code.
@License        Dual MIT/GPLv2

The contents of this file are subject to the MIT license as set out below.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 2 ("GPL") in which case the provisions
of GPL are applicable instead of those above.

If you wish to allow use of your version of this file only under the terms of
GPL, and not to allow others to use your version of this file under the terms
of the MIT license, indicate your decision by deleting the provisions above
and replace them with the notice and other provisions required by GPL as set
out in the file called "GPL-COPYING" included in this distribution. If you do
not delete the provisions above, a recipient may use your version of this file
under the terms of either the MIT license or GPL.

This License is also included in this distribution in the file called
"MIT-COPYING".

EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ /**************************************************************************/

#ifndef _DEVICEMEM_UTILS_H_
#define _DEVICEMEM_UTILS_H_

#include "devicemem.h"
#include "img_types.h"
#include "pvrsrv_error.h"
#include "pvr_debug.h"
#include "allocmem.h"
#include "ra.h"
#include "osfunc.h"
#include "lock.h"
#include "osmmap.h"

#define DEVMEM_HEAPNAME_MAXLENGTH 160

#if defined(DEVMEM_DEBUG) && defined(REFCOUNT_DEBUG)
#define DEVMEM_REFCOUNT_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_ERROR, __FILE__, __LINE__, fmt, __VA_ARGS__)
#else
#define DEVMEM_REFCOUNT_PRINT(fmt, ...)
#endif

/* If we need a "hMapping" but we don't have a server-side mapping, we poison
 * the entry with this value so that it's easily recognised in the debugger.
 * Note that this is potentially a valid handle, but then so is NULL, which is
 * no better, indeed worse, as it's not obvious in the debugger. The value
 * doesn't matter. We _never_ use it (and because it's valid, we never assert
 * it isn't this) but it's nice to have a value in the source code that we can
 * grep for if things go wrong.
 */
#define LACK_OF_MAPPING_POISON ((IMG_HANDLE)0x6116dead)
#define LACK_OF_RESERVATION_POISON ((IMG_HANDLE)0x7117dead)

#define DEVICEMEM_HISTORY_ALLOC_INDEX_NONE 0xFFFFFFFF

struct _DEVMEM_CONTEXT_ {

	SHARED_DEV_CONNECTION hDevConnection;

	/* Number of heaps that have been created in this context
	 * (regardless of whether they have allocations)
	 */
	IMG_UINT32 uiNumHeaps;

	/* Each "DEVMEM_CONTEXT" has a counterpart in the server, which
	 * is responsible for handling the mapping into device MMU.
	 * We have a handle to that here.
	 */
	IMG_HANDLE hDevMemServerContext;

	/* Number of automagically created heaps in this context,
	 *  i.e. those that are born at context creation time from the
	 * chosen "heap config" or "blueprint"
	 */
	IMG_UINT32 uiAutoHeapCount;

	/* Pointer to array of such heaps */
	struct _DEVMEM_HEAP_ **ppsAutoHeapArray;

	/* The cache line size for use when allocating memory,
	 * as it is not queryable on the client side
	 */
	IMG_UINT32 ui32CPUCacheLineSize;

	/* Private data handle for device specific data */
	IMG_HANDLE hPrivData;

	/* Memory allocated to be used for MCU fences */
	DEVMEM_MEMDESC *psMCUFenceMemDesc;
};


typedef enum
{
	DEVMEM_HEAP_TYPE_UNKNOWN = 0,
	DEVMEM_HEAP_TYPE_USER_MANAGED,
	DEVMEM_HEAP_TYPE_KERNEL_MANAGED,
	DEVMEM_HEAP_TYPE_RA_MANAGED,
}DEVMEM_HEAP_TYPE;

struct _DEVMEM_HEAP_ {
	/* Name of heap - for debug and lookup purposes. */
	IMG_CHAR *pszName;

	/* Number of live imports in the heap */
	ATOMIC_T hImportCount;

	/* Base address and size of heap, required by clients due to some
	 * requesters not being full range
	 */
	IMG_DEV_VIRTADDR sBaseAddress;
	DEVMEM_SIZE_T uiSize;

	/* The heap type, describing if the space is managed by the user or an RA
	 */
	DEVMEM_HEAP_TYPE eHeapType;

	/* This RA is for managing sub-allocations in virtual space. Two more
	 * RA's will be used under the Hood for managing the coarser allocation
	 * of virtual space from the heap, and also for managing the physical
	 * backing storage.
	 */
	RA_ARENA *psSubAllocRA;
	IMG_CHAR *pszSubAllocRAName;

	/* This RA is for the coarse allocation of virtual space from the heap */
	RA_ARENA *psQuantizedVMRA;
	IMG_CHAR *pszQuantizedVMRAName;

	/* We also need to store a copy of the quantum size in order to feed
	 * this down to the server.
	 */
	IMG_UINT32 uiLog2Quantum;

	/* Store a copy of the minimum import alignment */
	IMG_UINT32 uiLog2ImportAlignment;

	/* The relationship between tiled heap alignment and heap byte-stride
	 * (dependent on tiling mode, abstracted here)
	 */
	IMG_UINT32 uiLog2TilingStrideFactor;

	/* The parent memory context for this heap */
	struct _DEVMEM_CONTEXT_ *psCtx;

	/* Lock to protect this structure */
	POS_LOCK hLock;

	/* Each "DEVMEM_HEAP" has a counterpart in the server, which is
	 * responsible for handling the mapping into device MMU.
	 * We have a handle to that here.
	 */
	IMG_HANDLE hDevMemServerHeap;
};

typedef IMG_UINT32 DEVMEM_PROPERTIES_T;                  /*!< Typedef for Devicemem properties */
#define DEVMEM_PROPERTIES_EXPORTABLE         (1UL<<0)    /*!< Is it exportable? */
#define DEVMEM_PROPERTIES_IMPORTED           (1UL<<1)    /*!< Is it imported from another process? */
#define DEVMEM_PROPERTIES_SUBALLOCATABLE     (1UL<<2)    /*!< Is it suballocatable? */
#define DEVMEM_PROPERTIES_UNPINNED           (1UL<<3)    /*!< Is it currently pinned? */
#define DEVMEM_PROPERTIES_IMPORT_IS_ZEROED   (1UL<<4)    /*!< Is the memory fully zeroed? */
#define DEVMEM_PROPERTIES_IMPORT_IS_CLEAN    (1UL<<5)    /*!< Is the memory clean, i.e. not been used before? */
#define DEVMEM_PROPERTIES_SECURE             (1UL<<6)    /*!< Is it a special secure buffer? No CPU maps allowed! */
#define DEVMEM_PROPERTIES_IMPORT_IS_POISONED (1UL<<7)    /*!< Is the memory fully poisoned? */
#define DEVMEM_PROPERTIES_NO_CPU_MAPPING     (1UL<<8)    /* No CPU Mapping is allowed, RW attributes
                                                            are further derived from allocation memory flags */
#define DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE	 (1UL<<9)    /* No sparse resizing allowed, once a memory
                                                            layout is chosen, no change allowed later,
                                                            This includes pinning and unpinning */


typedef struct _DEVMEM_DEVICE_IMPORT_ {
	DEVMEM_HEAP *psHeap;            /*!< Heap this import is bound to */
	IMG_DEV_VIRTADDR sDevVAddr;     /*!< Device virtual address of the import */
	IMG_UINT32 ui32RefCount;        /*!< Refcount of the device virtual address */
	IMG_HANDLE hReservation;        /*!< Device memory reservation handle */
	IMG_HANDLE hMapping;            /*!< Device mapping handle */
	IMG_BOOL bMapped;               /*!< This is import mapped? */
	POS_LOCK hLock;                 /*!< Lock to protect the device import */
} DEVMEM_DEVICE_IMPORT;

typedef struct _DEVMEM_CPU_IMPORT_ {
	void *pvCPUVAddr;               /*!< CPU virtual address of the import */
	IMG_UINT32 ui32RefCount;        /*!< Refcount of the CPU virtual address */
	IMG_HANDLE hOSMMapData;         /*!< CPU mapping handle */
	POS_LOCK hLock;                 /*!< Lock to protect the CPU import */
} DEVMEM_CPU_IMPORT;

typedef struct _DEVMEM_IMPORT_ {
	SHARED_DEV_CONNECTION hDevConnection;
	IMG_DEVMEM_ALIGN_T uiAlign;         /*!< Alignment of the PMR */
	DEVMEM_SIZE_T uiSize;               /*!< Size of import */
	ATOMIC_T hRefCount;                 /*!< Refcount for this import */
	DEVMEM_PROPERTIES_T uiProperties;   /*!< Stores properties of an import like if
	                                         it is exportable, pinned or suballocatable */
	IMG_HANDLE hPMR;                    /*!< Handle to the PMR */
	DEVMEM_FLAGS_T uiFlags;             /*!< Flags for this import */
	POS_LOCK hLock;                     /*!< Lock to protect the import */

	DEVMEM_DEVICE_IMPORT sDeviceImport; /*!< Device specifics of the import */
	DEVMEM_CPU_IMPORT sCPUImport;       /*!< CPU specifics of the import */
} DEVMEM_IMPORT;

typedef struct _DEVMEM_DEVICE_MEMDESC_ {
	IMG_DEV_VIRTADDR sDevVAddr;     /*!< Device virtual address of the allocation */
	IMG_UINT32 ui32RefCount;        /*!< Refcount of the device virtual address */
	POS_LOCK hLock;                 /*!< Lock to protect device memdesc */
} DEVMEM_DEVICE_MEMDESC;

typedef struct _DEVMEM_CPU_MEMDESC_ {
	void *pvCPUVAddr;           /*!< CPU virtual address of the import */
	IMG_UINT32 ui32RefCount;    /*!< Refcount of the device CPU address */
	POS_LOCK hLock;             /*!< Lock to protect CPU memdesc */
} DEVMEM_CPU_MEMDESC;

struct _DEVMEM_MEMDESC_ {
	DEVMEM_IMPORT *psImport;                /*!< Import this memdesc is on */
	IMG_DEVMEM_OFFSET_T uiOffset;           /*!< Offset into import where our allocation starts */
	IMG_DEVMEM_SIZE_T uiAllocSize;          /*!< Size of the allocation */
	ATOMIC_T hRefCount;                     /*!< Refcount of the memdesc */
	POS_LOCK hLock;                         /*!< Lock to protect memdesc */
	IMG_HANDLE hPrivData;

	DEVMEM_DEVICE_MEMDESC sDeviceMemDesc;   /*!< Device specifics of the memdesc */
	DEVMEM_CPU_MEMDESC sCPUMemDesc;         /*!< CPU specifics of the memdesc */

	IMG_CHAR szText[DEVMEM_ANNOTATION_MAX_LEN]; /*!< Annotation for this memdesc */

	IMG_UINT32 ui32AllocationIndex;

#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
	IMG_HANDLE hRIHandle;                   /*!< Handle to RI information */
#endif
};

/* The physical descriptor used to store handles and information of device
 * physical allocations.
 */
struct _DEVMEMX_PHYS_MEMDESC_ {
	IMG_UINT32 uiNumPages;                  /*!< Number of pages that the import has*/
	IMG_UINT32 uiLog2PageSize;              /*!< Page size */
	ATOMIC_T hRefCount;                     /*!< Refcount of the memdesc */
	DEVMEM_FLAGS_T uiFlags;                 /*!< Flags for this import */
	IMG_HANDLE hPMR;                        /*!< Handle to the PMR */
	DEVMEM_CPU_IMPORT sCPUImport;           /*!< CPU specifics of the memdesc */
	DEVMEM_BRIDGE_HANDLE hBridge;           /*!< Bridge connection for the server */
};

/* The virtual descriptor used to store handles and information of a device
 * virtual range and the mappings to it.
 */
struct _DEVMEMX_VIRT_MEMDESC_ {
	IMG_UINT32 uiNumPages;                  /*!< Number of pages that the import has*/
	DEVMEM_FLAGS_T uiFlags;                 /*!< Flags for this import */
	DEVMEMX_PHYSDESC **apsPhysDescTable;    /*!< Table to store links to physical descs */
	DEVMEM_DEVICE_IMPORT sDeviceImport;     /*!< Device specifics of the memdesc */

	IMG_CHAR szText[DEVMEM_ANNOTATION_MAX_LEN]; /*!< Annotation for this virt memdesc */
	IMG_UINT32 ui32AllocationIndex;         /*!< To track mappings in this range */

#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
	IMG_HANDLE hRIHandle;                   /*!< Handle to RI information */
#endif
};

#define DEVICEMEM_UTILS_NO_ADDRESS 0

/******************************************************************************
@Function       _DevmemValidateParams
@Description    Check if flags are conflicting and if align is a size multiple.

@Input          uiSize      Size of the import.
@Input          uiAlign     Alignment of the import.
@Input          puiFlags    Pointer to the flags for the import.
@return         PVRSRV_ERROR
******************************************************************************/
PVRSRV_ERROR _DevmemValidateParams(IMG_DEVMEM_SIZE_T uiSize,
                                   IMG_DEVMEM_ALIGN_T uiAlign,
                                   DEVMEM_FLAGS_T *puiFlags);

/******************************************************************************
@Function       _DevmemImportStructAlloc
@Description    Allocates memory for an import struct. Does not allocate a PMR!
                Create locks for CPU and Devmem mappings.

@Input          hDevConnection  Connection to use for calls from the import.
@Input          ppsImport       The import to allocate.
@return         PVRSRV_ERROR
******************************************************************************/
PVRSRV_ERROR _DevmemImportStructAlloc(SHARED_DEV_CONNECTION hDevConnection,
                                      DEVMEM_IMPORT **ppsImport);

/******************************************************************************
@Function       _DevmemImportStructInit
@Description    Initialises the import struct with the given parameters.
                Set it's refcount to 1!

@Input          psImport     The import to initialise.
@Input          uiSize       Size of the import.
@Input          uiAlign      Alignment of allocations in the import.
@Input          uiMapFlags
@Input          hPMR         Reference to the PMR of this import struct.
@Input          uiProperties Properties of the import. Is it exportable,
                              imported, suballocatable, unpinned?
******************************************************************************/
void _DevmemImportStructInit(DEVMEM_IMPORT *psImport,
                             IMG_DEVMEM_SIZE_T uiSize,
                             IMG_DEVMEM_ALIGN_T uiAlign,
                             PVRSRV_MEMALLOCFLAGS_T uiMapFlags,
                             IMG_HANDLE hPMR,
                             DEVMEM_PROPERTIES_T uiProperties);

/******************************************************************************
@Function       _DevmemImportStructDevMap
@Description    NEVER call after the last _DevmemMemDescRelease()
                Maps the PMR referenced by the import struct to the device's
                virtual address space.
                Does nothing but increase the cpu mapping refcount if the
                import struct was already mapped.

@Input          psHeap    The heap to map to.
@Input          bMap      Caller can choose if the import should be really
                          mapped in the page tables or if just a virtual range
                          should be reserved and the refcounts increased.
@Input          psImport  The import we want to map.
@Input          uiOptionalMapAddress  An optional address to map to.
                                      Pass DEVICEMEM_UTILS_NOADDRESS if not used.
@return         PVRSRV_ERROR
******************************************************************************/
PVRSRV_ERROR _DevmemImportStructDevMap(DEVMEM_HEAP *psHeap,
                                       IMG_BOOL bMap,
                                       DEVMEM_IMPORT *psImport,
                                       IMG_UINT64 uiOptionalMapAddress);

/******************************************************************************
@Function       _DevmemImportStructDevUnmap
@Description    Unmaps the PMR referenced by the import struct from the
                device's virtual address space.
                If this was not the last remaining CPU mapping on the import
                struct only the cpu mapping refcount is decreased.
******************************************************************************/
void _DevmemImportStructDevUnmap(DEVMEM_IMPORT *psImport);

/******************************************************************************
@Function       _DevmemImportStructCPUMap
@Description    NEVER call after the last _DevmemMemDescRelease()
                Maps the PMR referenced by the import struct to the CPU's
                virtual address space.
                Does nothing but increase the cpu mapping refcount if the
                import struct was already mapped.
@return         PVRSRV_ERROR
******************************************************************************/
PVRSRV_ERROR _DevmemImportStructCPUMap(DEVMEM_IMPORT *psImport);

/******************************************************************************
@Function       _DevmemImportStructCPUUnmap
@Description    Unmaps the PMR referenced by the import struct from the CPU's
                virtual address space.
                If this was not the last remaining CPU mapping on the import
                struct only the cpu mapping refcount is decreased.
******************************************************************************/
void _DevmemImportStructCPUUnmap(DEVMEM_IMPORT *psImport);


/******************************************************************************
@Function       _DevmemImportStructAcquire
@Description    Acquire an import struct by increasing it's refcount.
******************************************************************************/
void _DevmemImportStructAcquire(DEVMEM_IMPORT *psImport);

/******************************************************************************
@Function       _DevmemImportStructRelease
@Description    Reduces the refcount of the import struct.
                Destroys the import in the case it was the last reference.
                Destroys underlying PMR if this import was the last reference
                to it.
@return         A boolean to signal if the import was destroyed. True = yes.
******************************************************************************/
IMG_BOOL _DevmemImportStructRelease(DEVMEM_IMPORT *psImport);

/******************************************************************************
@Function       _DevmemImportDiscard
@Description    Discard a created, but unitilised import structure.
                This must only be called before _DevmemImportStructInit
                after which _DevmemImportStructRelease must be used to
                "free" the import structure.
******************************************************************************/
void _DevmemImportDiscard(DEVMEM_IMPORT *psImport);

/******************************************************************************
@Function       _DevmemMemDescAlloc
@Description    Allocates a MemDesc and create it's various locks.
                Zero the allocated memory.
@return         PVRSRV_ERROR
******************************************************************************/
PVRSRV_ERROR _DevmemMemDescAlloc(DEVMEM_MEMDESC **ppsMemDesc);

/******************************************************************************
@Function       _DevmemMemDescInit
@Description    Sets the given offset and import struct fields in the MemDesc.
                Initialises refcount to 1 and other values to 0.

@Input          psMemDesc    MemDesc to initialise.
@Input          uiOffset     Offset in the import structure.
@Input          psImport     Import the MemDesc is on.
@Input          uiAllocSize  Size of the allocation
******************************************************************************/
void _DevmemMemDescInit(DEVMEM_MEMDESC *psMemDesc,
						IMG_DEVMEM_OFFSET_T uiOffset,
						DEVMEM_IMPORT *psImport,
						IMG_DEVMEM_SIZE_T uiAllocSize);

/******************************************************************************
@Function       _DevmemMemDescAcquire
@Description    Acquires the MemDesc by increasing it's refcount.
******************************************************************************/
void _DevmemMemDescAcquire(DEVMEM_MEMDESC *psMemDesc);

/******************************************************************************
@Function       _DevmemMemDescRelease
@Description    Releases the MemDesc by reducing it's refcount.
                Destroy the MemDesc if it's recount is 0.
                Destroy the import struct the MemDesc is on if that was the
                last MemDesc on the import, probably following the destruction
                of the underlying PMR.
@return         A boolean to signal if the MemDesc was destroyed. True = yes.
******************************************************************************/
IMG_BOOL _DevmemMemDescRelease(DEVMEM_MEMDESC *psMemDesc);

/******************************************************************************
@Function       _DevmemMemDescDiscard
@Description    Discard a created, but uninitialised MemDesc structure.
                This must only be called before _DevmemMemDescInit after
                which _DevmemMemDescRelease must be used to "free" the
                MemDesc structure.
******************************************************************************/
void _DevmemMemDescDiscard(DEVMEM_MEMDESC *psMemDesc);

#endif /* _DEVICEMEM_UTILS_H_ */