| /* -*- mode: c; indent-tabs-mode: t; c-basic-offset: 8; tab-width: 8 -*- */ |
| /* vi: set ts=8 sw=8 sts=8: */ |
| /*************************************************************************/ /*! |
| @File |
| @Title Linux buffer sync interface |
| @Codingstyle LinuxKernel |
| @Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved |
| @License Dual MIT/GPLv2 |
| |
| The contents of this file are subject to the MIT license as set out below. |
| |
| Permission is hereby granted, free of charge, to any person obtaining a copy |
| of this software and associated documentation files (the "Software"), to deal |
| in the Software without restriction, including without limitation the rights |
| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| copies of the Software, and to permit persons to whom the Software is |
| furnished to do so, subject to the following conditions: |
| |
| The above copyright notice and this permission notice shall be included in |
| all copies or substantial portions of the Software. |
| |
| Alternatively, the contents of this file may be used under the terms of |
| the GNU General Public License Version 2 ("GPL") in which case the provisions |
| of GPL are applicable instead of those above. |
| |
| If you wish to allow use of your version of this file only under the terms of |
| GPL, and not to allow others to use your version of this file under the terms |
| of the MIT license, indicate your decision by deleting the provisions above |
| and replace them with the notice and other provisions required by GPL as set |
| out in the file called "GPL-COPYING" included in this distribution. If you do |
| not delete the provisions above, a recipient may use your version of this file |
| under the terms of either the MIT license or GPL. |
| |
| This License is also included in this distribution in the file called |
| "MIT-COPYING". |
| |
| EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS |
| PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING |
| BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR |
| PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR |
| COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER |
| IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| */ /**************************************************************************/ |
| |
| #include <linux/dma-buf.h> |
| #include <linux/reservation.h> |
| |
| #include "services_kernel_client.h" |
| #include "pvr_buffer_sync.h" |
| #include "pvr_buffer_sync_shared.h" |
| #include "pvr_drv.h" |
| #include "pvr_fence.h" |
| |
| |
| struct pvr_buffer_sync_context { |
| #if !defined(PVRSRV_USE_BRIDGE_LOCK) |
| struct mutex ctx_lock; |
| #endif |
| struct pvr_fence_context *fence_ctx; |
| struct ww_acquire_ctx acquire_ctx; |
| }; |
| |
| struct pvr_buffer_sync_check_data { |
| struct dma_fence_cb base; |
| |
| u32 nr_fences; |
| struct pvr_fence **fences; |
| }; |
| |
| struct pvr_buffer_sync_append_data { |
| struct pvr_buffer_sync_context *ctx; |
| |
| u32 nr_pmrs; |
| struct _PMR_ **pmrs; |
| u32 *pmr_flags; |
| |
| struct pvr_fence *update_fence; |
| struct pvr_buffer_sync_check_data *check_data; |
| }; |
| |
| |
| static struct reservation_object * |
| pmr_reservation_object_get(struct _PMR_ *pmr) |
| { |
| struct dma_buf *dmabuf; |
| |
| dmabuf = PhysmemGetDmaBuf(pmr); |
| if (dmabuf) |
| return dmabuf->resv; |
| |
| return NULL; |
| } |
| |
| static int |
| pvr_buffer_sync_pmrs_lock(struct pvr_buffer_sync_context *ctx, |
| u32 nr_pmrs, |
| struct _PMR_ **pmrs) |
| { |
| struct reservation_object *resv, *cresv = NULL, *lresv = NULL; |
| int i, err; |
| struct ww_acquire_ctx *acquire_ctx = &ctx->acquire_ctx; |
| |
| #if !defined(PVRSRV_USE_BRIDGE_LOCK) |
| mutex_lock(&ctx->ctx_lock); |
| #endif |
| |
| ww_acquire_init(acquire_ctx, &reservation_ww_class); |
| retry: |
| for (i = 0; i < nr_pmrs; i++) { |
| resv = pmr_reservation_object_get(pmrs[i]); |
| if (!resv) { |
| pr_err("%s: Failed to get reservation object from pmr %p\n", |
| __func__, pmrs[i]); |
| err = -EINVAL; |
| goto fail; |
| } |
| |
| if (resv != lresv) { |
| err = ww_mutex_lock_interruptible(&resv->lock, |
| acquire_ctx); |
| if (err) { |
| cresv = (err == -EDEADLK) ? resv : NULL; |
| goto fail; |
| } |
| } else { |
| lresv = NULL; |
| } |
| } |
| |
| ww_acquire_done(acquire_ctx); |
| |
| return 0; |
| |
| fail: |
| while (i--) { |
| resv = pmr_reservation_object_get(pmrs[i]); |
| if (WARN_ON_ONCE(!resv)) |
| continue; |
| ww_mutex_unlock(&resv->lock); |
| } |
| |
| if (lresv) |
| ww_mutex_unlock(&lresv->lock); |
| |
| if (cresv) { |
| err = ww_mutex_lock_slow_interruptible(&cresv->lock, |
| acquire_ctx); |
| if (!err) { |
| lresv = cresv; |
| cresv = NULL; |
| goto retry; |
| } |
| } |
| |
| ww_acquire_fini(acquire_ctx); |
| |
| #if !defined(PVRSRV_USE_BRIDGE_LOCK) |
| mutex_unlock(&ctx->ctx_lock); |
| #endif |
| return err; |
| } |
| |
| static void |
| pvr_buffer_sync_pmrs_unlock(struct pvr_buffer_sync_context *ctx, |
| u32 nr_pmrs, |
| struct _PMR_ **pmrs) |
| { |
| struct reservation_object *resv; |
| int i; |
| struct ww_acquire_ctx *acquire_ctx = &ctx->acquire_ctx; |
| |
| for (i = 0; i < nr_pmrs; i++) { |
| resv = pmr_reservation_object_get(pmrs[i]); |
| if (WARN_ON_ONCE(!resv)) |
| continue; |
| ww_mutex_unlock(&resv->lock); |
| } |
| |
| ww_acquire_fini(acquire_ctx); |
| |
| #if !defined(PVRSRV_USE_BRIDGE_LOCK) |
| mutex_unlock(&ctx->ctx_lock); |
| #endif |
| } |
| |
| static u32 |
| pvr_buffer_sync_pmrs_fence_count(u32 nr_pmrs, struct _PMR_ **pmrs, |
| u32 *pmr_flags) |
| { |
| struct reservation_object *resv; |
| struct reservation_object_list *resv_list; |
| struct dma_fence *fence; |
| u32 fence_count = 0; |
| bool exclusive; |
| int i; |
| |
| for (i = 0; i < nr_pmrs; i++) { |
| exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE); |
| |
| resv = pmr_reservation_object_get(pmrs[i]); |
| if (WARN_ON_ONCE(!resv)) |
| continue; |
| |
| resv_list = reservation_object_get_list(resv); |
| fence = reservation_object_get_excl(resv); |
| |
| if (fence && |
| (!exclusive || !resv_list || !resv_list->shared_count)) |
| fence_count++; |
| |
| if (exclusive && resv_list) |
| fence_count += resv_list->shared_count; |
| } |
| |
| return fence_count; |
| } |
| |
| static struct pvr_buffer_sync_check_data * |
| pvr_buffer_sync_check_fences_create(struct pvr_fence_context *fence_ctx, |
| PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx, |
| u32 nr_pmrs, |
| struct _PMR_ **pmrs, |
| u32 *pmr_flags) |
| { |
| struct pvr_buffer_sync_check_data *data; |
| struct reservation_object *resv; |
| struct reservation_object_list *resv_list; |
| struct dma_fence *fence; |
| u32 fence_count; |
| bool exclusive; |
| int i, j; |
| int err; |
| |
| data = kzalloc(sizeof(*data), GFP_KERNEL); |
| if (!data) |
| return NULL; |
| |
| fence_count = pvr_buffer_sync_pmrs_fence_count(nr_pmrs, pmrs, |
| pmr_flags); |
| if (fence_count) { |
| data->fences = kcalloc(fence_count, sizeof(*data->fences), |
| GFP_KERNEL); |
| if (!data->fences) |
| goto err_check_data_free; |
| } |
| |
| for (i = 0; i < nr_pmrs; i++) { |
| resv = pmr_reservation_object_get(pmrs[i]); |
| if (WARN_ON_ONCE(!resv)) |
| continue; |
| |
| exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE); |
| if (!exclusive) { |
| err = reservation_object_reserve_shared(resv |
| #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)) |
| , 1 |
| #endif |
| ); |
| if (err) |
| goto err_destroy_fences; |
| } |
| |
| resv_list = reservation_object_get_list(resv); |
| fence = reservation_object_get_excl(resv); |
| |
| if (fence && |
| (!exclusive || !resv_list || !resv_list->shared_count)) { |
| data->fences[data->nr_fences++] = |
| pvr_fence_create_from_fence(fence_ctx, |
| sync_checkpoint_ctx, |
| fence, |
| PVRSRV_NO_FENCE, |
| "exclusive check fence"); |
| if (!data->fences[data->nr_fences - 1]) { |
| data->nr_fences--; |
| PVR_FENCE_TRACE(fence, |
| "waiting on exclusive fence\n"); |
| WARN_ON(dma_fence_wait(fence, true) <= 0); |
| } |
| } |
| |
| if (exclusive && resv_list) { |
| for (j = 0; j < resv_list->shared_count; j++) { |
| fence = rcu_dereference_protected(resv_list->shared[j], |
| reservation_object_held(resv)); |
| data->fences[data->nr_fences++] = |
| pvr_fence_create_from_fence(fence_ctx, |
| sync_checkpoint_ctx, |
| fence, |
| PVRSRV_NO_FENCE, |
| "check fence"); |
| if (!data->fences[data->nr_fences - 1]) { |
| data->nr_fences--; |
| PVR_FENCE_TRACE(fence, |
| "waiting on non-exclusive fence\n"); |
| WARN_ON(dma_fence_wait(fence, true) <= 0); |
| } |
| } |
| } |
| } |
| |
| WARN_ON((i != nr_pmrs) || (data->nr_fences != fence_count)); |
| |
| return data; |
| |
| err_destroy_fences: |
| for (i = 0; i < data->nr_fences; i++) |
| pvr_fence_destroy(data->fences[i]); |
| kfree(data->fences); |
| err_check_data_free: |
| kfree(data); |
| return NULL; |
| } |
| |
| static void |
| pvr_buffer_sync_check_fences_destroy(struct pvr_buffer_sync_check_data *data) |
| { |
| int i; |
| |
| for (i = 0; i < data->nr_fences; i++) |
| pvr_fence_destroy(data->fences[i]); |
| |
| kfree(data->fences); |
| kfree(data); |
| } |
| |
| struct pvr_buffer_sync_context * |
| pvr_buffer_sync_context_create(struct device *dev, const char *name) |
| { |
| struct drm_device *ddev = dev_get_drvdata(dev); |
| struct pvr_drm_private *priv = ddev->dev_private; |
| struct pvr_buffer_sync_context *ctx; |
| int err; |
| |
| ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); |
| if (!ctx) { |
| err = -ENOMEM; |
| goto err_exit; |
| } |
| |
| ctx->fence_ctx = pvr_fence_context_create(priv->dev_node, |
| priv->fence_status_wq, |
| name); |
| if (!ctx->fence_ctx) { |
| err = -ENOMEM; |
| goto err_free_ctx; |
| } |
| |
| #if !defined(PVRSRV_USE_BRIDGE_LOCK) |
| mutex_init(&ctx->ctx_lock); |
| #endif |
| |
| return ctx; |
| |
| err_free_ctx: |
| kfree(ctx); |
| err_exit: |
| return ERR_PTR(err); |
| } |
| |
| void |
| pvr_buffer_sync_context_destroy(struct pvr_buffer_sync_context *ctx) |
| { |
| pvr_fence_context_destroy(ctx->fence_ctx); |
| kfree(ctx); |
| } |
| |
| int |
| pvr_buffer_sync_resolve_and_create_fences(struct pvr_buffer_sync_context *ctx, |
| PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx, |
| u32 nr_pmrs, |
| struct _PMR_ **pmrs, |
| u32 *pmr_flags, |
| u32 *nr_fence_checkpoints_out, |
| PSYNC_CHECKPOINT **fence_checkpoints_out, |
| PSYNC_CHECKPOINT *update_checkpoints_out, |
| struct pvr_buffer_sync_append_data **data_out) |
| { |
| struct pvr_buffer_sync_append_data *data; |
| PSYNC_CHECKPOINT *fence_checkpoints; |
| const size_t data_size = sizeof(*data); |
| const size_t pmrs_size = sizeof(*pmrs) * nr_pmrs; |
| const size_t pmr_flags_size = sizeof(*pmr_flags) * nr_pmrs; |
| int i; |
| int j; |
| int err; |
| |
| if (unlikely((nr_pmrs && !(pmrs && pmr_flags)) || |
| !nr_fence_checkpoints_out || !fence_checkpoints_out || |
| !update_checkpoints_out)) |
| return -EINVAL; |
| |
| for (i = 0; i < nr_pmrs; i++) { |
| if (unlikely(!(pmr_flags[i] & PVR_BUFFER_FLAG_MASK))) { |
| pr_err("%s: Invalid flags %#08x for pmr %p\n", |
| __func__, pmr_flags[i], pmrs[i]); |
| return -EINVAL; |
| } |
| } |
| |
| #if defined(NO_HARDWARE) |
| /* |
| * For NO_HARDWARE there's no checking or updating of sync checkpoints |
| * which means SW waits on our fences will cause a deadlock (since they |
| * will never be signalled). Avoid this by not creating any fences. |
| */ |
| nr_pmrs = 0; |
| #endif |
| |
| if (!nr_pmrs) { |
| *nr_fence_checkpoints_out = 0; |
| *fence_checkpoints_out = NULL; |
| *update_checkpoints_out = NULL; |
| *data_out = NULL; |
| |
| return 0; |
| } |
| |
| data = kzalloc(data_size + pmrs_size + pmr_flags_size, GFP_KERNEL); |
| if (unlikely(!data)) |
| return -ENOMEM; |
| |
| data->ctx = ctx; |
| data->pmrs = (struct _PMR_ **)((char *)data + data_size); |
| data->pmr_flags = (u32 *)((char *)data->pmrs + pmrs_size); |
| |
| /* |
| * It's expected that user space will provide a set of unique PMRs |
| * but, as a PMR can have multiple handles, it's still possible to |
| * end up here with duplicates. Take this opportunity to filter out |
| * any remaining duplicates (updating flags when necessary) before |
| * trying to process them further. |
| */ |
| for (i = 0; i < nr_pmrs; i++) { |
| for (j = 0; j < data->nr_pmrs; j++) { |
| if (data->pmrs[j] == pmrs[i]) { |
| data->pmr_flags[j] |= pmr_flags[i]; |
| break; |
| } |
| } |
| |
| if (j == data->nr_pmrs) { |
| data->pmrs[j] = pmrs[i]; |
| data->pmr_flags[j] = pmr_flags[i]; |
| data->nr_pmrs++; |
| } |
| } |
| |
| err = pvr_buffer_sync_pmrs_lock(ctx, data->nr_pmrs, data->pmrs); |
| if (unlikely(err)) { |
| pr_err("%s: failed to lock pmrs (errno=%d)\n", |
| __func__, err); |
| goto err_free_data; |
| } |
| |
| /* create the check data */ |
| data->check_data = pvr_buffer_sync_check_fences_create(ctx->fence_ctx, |
| sync_checkpoint_ctx, |
| data->nr_pmrs, |
| data->pmrs, |
| data->pmr_flags); |
| if (unlikely(!data->check_data)) { |
| err = -ENOMEM; |
| goto err_pmrs_unlock; |
| } |
| |
| fence_checkpoints = kcalloc(data->check_data->nr_fences, |
| sizeof(*fence_checkpoints), |
| GFP_KERNEL); |
| if (fence_checkpoints) { |
| pvr_fence_get_checkpoints(data->check_data->fences, |
| data->check_data->nr_fences, |
| fence_checkpoints); |
| } else { |
| if (unlikely(data->check_data->nr_fences)) { |
| err = -ENOMEM; |
| goto err_free_check_data; |
| } |
| } |
| |
| /* create the update fence */ |
| data->update_fence = pvr_fence_create(ctx->fence_ctx, |
| sync_checkpoint_ctx, |
| SYNC_CHECKPOINT_FOREIGN_CHECKPOINT, "update fence"); |
| if (unlikely(!data->update_fence)) { |
| err = -ENOMEM; |
| goto err_free_fence_checkpoints; |
| } |
| |
| /* |
| * We need to clean up the fences once the HW has finished with them. |
| * We can do this using fence callbacks. However, instead of adding a |
| * callback to every fence, which would result in more work, we can |
| * simply add one to the update fence since this will be the last fence |
| * to be signalled. This callback can do all the necessary clean up. |
| * |
| * Note: we take an additional reference on the update fence in case |
| * it signals before we can add it to a reservation object. |
| */ |
| PVR_FENCE_TRACE(&data->update_fence->base, |
| "create fence calling dma_fence_get\n"); |
| dma_fence_get(&data->update_fence->base); |
| |
| *nr_fence_checkpoints_out = data->check_data->nr_fences; |
| *fence_checkpoints_out = fence_checkpoints; |
| *update_checkpoints_out = pvr_fence_get_checkpoint(data->update_fence); |
| *data_out = data; |
| |
| return err; |
| |
| err_free_fence_checkpoints: |
| kfree(fence_checkpoints); |
| err_free_check_data: |
| pvr_buffer_sync_check_fences_destroy(data->check_data); |
| err_pmrs_unlock: |
| pvr_buffer_sync_pmrs_unlock(ctx, data->nr_pmrs, data->pmrs); |
| err_free_data: |
| kfree(data); |
| return err; |
| } |
| |
| void |
| pvr_buffer_sync_kick_succeeded(struct pvr_buffer_sync_append_data *data) |
| { |
| struct reservation_object *resv; |
| int i; |
| |
| dma_fence_enable_sw_signaling(&data->update_fence->base); |
| |
| for (i = 0; i < data->nr_pmrs; i++) { |
| resv = pmr_reservation_object_get(data->pmrs[i]); |
| if (WARN_ON_ONCE(!resv)) |
| continue; |
| |
| if (data->pmr_flags[i] & PVR_BUFFER_FLAG_WRITE) { |
| PVR_FENCE_TRACE(&data->update_fence->base, |
| "added exclusive fence (%s) to resv %p\n", |
| data->update_fence->name, resv); |
| reservation_object_add_excl_fence(resv, |
| &data->update_fence->base); |
| } else if (data->pmr_flags[i] & PVR_BUFFER_FLAG_READ) { |
| PVR_FENCE_TRACE(&data->update_fence->base, |
| "added non-exclusive fence (%s) to resv %p\n", |
| data->update_fence->name, resv); |
| reservation_object_add_shared_fence(resv, |
| &data->update_fence->base); |
| } |
| } |
| |
| /* |
| * Now that the fence has been added to the necessary |
| * reservation objects we can safely drop the extra reference |
| * we took in pvr_buffer_sync_resolve_and_create_fences(). |
| */ |
| dma_fence_put(&data->update_fence->base); |
| pvr_buffer_sync_pmrs_unlock(data->ctx, data->nr_pmrs, |
| data->pmrs); |
| |
| /* destroy the check fences */ |
| pvr_buffer_sync_check_fences_destroy(data->check_data); |
| /* destroy the update fence */ |
| pvr_fence_destroy(data->update_fence); |
| |
| /* free the append data */ |
| kfree(data); |
| } |
| |
| void |
| pvr_buffer_sync_kick_failed(struct pvr_buffer_sync_append_data *data) |
| { |
| |
| /* drop the extra reference we took on the update fence in |
| * pvr_buffer_sync_resolve_and_create_fences(). |
| */ |
| dma_fence_put(&data->update_fence->base); |
| |
| if (data->nr_pmrs > 0) |
| pvr_buffer_sync_pmrs_unlock(data->ctx, data->nr_pmrs, |
| data->pmrs); |
| |
| /* destroy the check fences */ |
| pvr_buffer_sync_check_fences_destroy(data->check_data); |
| /* destroy the update fence */ |
| pvr_fence_destroy(data->update_fence); |
| |
| /* free the append data */ |
| kfree(data); |
| } |
| |
| #if defined(PVRSRV_SYNC_CHECKPOINT_CCB) |
| enum tag_img_bool |
| pvr_buffer_sync_checkpoint_ufo_has_signalled(u32 fwaddr, u32 value) |
| { |
| return pvr_fence_checkpoint_ufo_has_signalled(fwaddr, value); |
| } |
| |
| void |
| pvr_buffer_sync_check_state(void) |
| { |
| pvr_fence_check_state(); |
| } |
| #endif /* defined(PVRSRV_SYNC_CHECKPOINT_CCB) */ |