diff options
Diffstat (limited to 'drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c')
| -rw-r--r-- | drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 406 |
1 files changed, 406 insertions, 0 deletions
diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c new file mode 100644 index 000000000000..7eef23a00d77 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_gt_tlb_invalidation.h" + +#include "abi/guc_actions_abi.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_trace.h" + +#define TLB_TIMEOUT (HZ / 4) + +static void xe_gt_tlb_fence_timeout(struct work_struct *work) +{ + struct xe_gt *gt = container_of(work, struct xe_gt, + tlb_invalidation.fence_tdr.work); + struct xe_gt_tlb_invalidation_fence *fence, *next; + + spin_lock_irq(>->tlb_invalidation.pending_lock); + list_for_each_entry_safe(fence, next, + >->tlb_invalidation.pending_fences, link) { + s64 since_inval_ms = ktime_ms_delta(ktime_get(), + fence->invalidation_time); + + if (msecs_to_jiffies(since_inval_ms) < TLB_TIMEOUT) + break; + + trace_xe_gt_tlb_invalidation_fence_timeout(fence); + drm_err(>_to_xe(gt)->drm, "gt%d: TLB invalidation fence timeout, seqno=%d recv=%d", + gt->info.id, fence->seqno, gt->tlb_invalidation.seqno_recv); + + list_del(&fence->link); + fence->base.error = -ETIME; + dma_fence_signal(&fence->base); + dma_fence_put(&fence->base); + } + if (!list_empty(>->tlb_invalidation.pending_fences)) + queue_delayed_work(system_wq, + >->tlb_invalidation.fence_tdr, + TLB_TIMEOUT); + spin_unlock_irq(>->tlb_invalidation.pending_lock); +} + +/** + * xe_gt_tlb_invalidation_init - Initialize GT TLB invalidation state + * @gt: graphics tile + * + * Initialize GT TLB invalidation state, purely software initialization, should + * be called once during driver load. + * + * Return: 0 on success, negative error code on error. + */ +int xe_gt_tlb_invalidation_init(struct xe_gt *gt) +{ + gt->tlb_invalidation.seqno = 1; + INIT_LIST_HEAD(>->tlb_invalidation.pending_fences); + spin_lock_init(>->tlb_invalidation.pending_lock); + spin_lock_init(>->tlb_invalidation.lock); + gt->tlb_invalidation.fence_context = dma_fence_context_alloc(1); + INIT_DELAYED_WORK(>->tlb_invalidation.fence_tdr, + xe_gt_tlb_fence_timeout); + + return 0; +} + +static void +__invalidation_fence_signal(struct xe_gt_tlb_invalidation_fence *fence) +{ + trace_xe_gt_tlb_invalidation_fence_signal(fence); + dma_fence_signal(&fence->base); + dma_fence_put(&fence->base); +} + +static void +invalidation_fence_signal(struct xe_gt_tlb_invalidation_fence *fence) +{ + list_del(&fence->link); + __invalidation_fence_signal(fence); +} + +/** + * xe_gt_tlb_invalidation_reset - Initialize GT TLB invalidation reset + * @gt: graphics tile + * + * Signal any pending invalidation fences, should be called during a GT reset + */ +void xe_gt_tlb_invalidation_reset(struct xe_gt *gt) +{ + struct xe_gt_tlb_invalidation_fence *fence, *next; + struct xe_guc *guc = >->uc.guc; + int pending_seqno; + + /* + * CT channel is already disabled at this point. No new TLB requests can + * appear. + */ + + mutex_lock(>->uc.guc.ct.lock); + spin_lock_irq(>->tlb_invalidation.pending_lock); + cancel_delayed_work(>->tlb_invalidation.fence_tdr); + /* + * We might have various kworkers waiting for TLB flushes to complete + * which are not tracked with an explicit TLB fence, however at this + * stage that will never happen since the CT is already disabled, so + * make sure we signal them here under the assumption that we have + * completed a full GT reset. + */ + if (gt->tlb_invalidation.seqno == 1) + pending_seqno = TLB_INVALIDATION_SEQNO_MAX - 1; + else + pending_seqno = gt->tlb_invalidation.seqno - 1; + WRITE_ONCE(gt->tlb_invalidation.seqno_recv, pending_seqno); + wake_up_all(&guc->ct.wq); + + list_for_each_entry_safe(fence, next, + >->tlb_invalidation.pending_fences, link) + invalidation_fence_signal(fence); + spin_unlock_irq(>->tlb_invalidation.pending_lock); + mutex_unlock(>->uc.guc.ct.lock); +} + +static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno) +{ + int seqno_recv = READ_ONCE(gt->tlb_invalidation.seqno_recv); + + if (seqno - seqno_recv < -(TLB_INVALIDATION_SEQNO_MAX / 2)) + return false; + + if (seqno - seqno_recv > (TLB_INVALIDATION_SEQNO_MAX / 2)) + return true; + + return seqno_recv >= seqno; +} + +static int send_tlb_invalidation(struct xe_guc *guc, + struct xe_gt_tlb_invalidation_fence *fence, + u32 *action, int len) +{ + struct xe_gt *gt = guc_to_gt(guc); + int seqno; + int ret; + + /* + * XXX: The seqno algorithm relies on TLB invalidation being processed + * in order which they currently are, if that changes the algorithm will + * need to be updated. + */ + + mutex_lock(&guc->ct.lock); + seqno = gt->tlb_invalidation.seqno; + if (fence) { + fence->seqno = seqno; + trace_xe_gt_tlb_invalidation_fence_send(fence); + } + action[1] = seqno; + ret = xe_guc_ct_send_locked(&guc->ct, action, len, + G2H_LEN_DW_TLB_INVALIDATE, 1); + if (!ret && fence) { + spin_lock_irq(>->tlb_invalidation.pending_lock); + /* + * We haven't actually published the TLB fence as per + * pending_fences, but in theory our seqno could have already + * been written as we acquired the pending_lock. In such a case + * we can just go ahead and signal the fence here. + */ + if (tlb_invalidation_seqno_past(gt, seqno)) { + __invalidation_fence_signal(fence); + } else { + fence->invalidation_time = ktime_get(); + list_add_tail(&fence->link, + >->tlb_invalidation.pending_fences); + + if (list_is_singular(>->tlb_invalidation.pending_fences)) + queue_delayed_work(system_wq, + >->tlb_invalidation.fence_tdr, + TLB_TIMEOUT); + } + spin_unlock_irq(>->tlb_invalidation.pending_lock); + } else if (ret < 0 && fence) { + __invalidation_fence_signal(fence); + } + if (!ret) { + gt->tlb_invalidation.seqno = (gt->tlb_invalidation.seqno + 1) % + TLB_INVALIDATION_SEQNO_MAX; + if (!gt->tlb_invalidation.seqno) + gt->tlb_invalidation.seqno = 1; + ret = seqno; + } + mutex_unlock(&guc->ct.lock); + + return ret; +} + +#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \ + XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \ + XE_GUC_TLB_INVAL_FLUSH_CACHE) + +/** + * xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC + * @gt: graphics tile + * + * Issue a TLB invalidation for the GuC. Completion of TLB is asynchronous and + * caller can use seqno + xe_gt_tlb_invalidation_wait to wait for completion. + * + * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success, + * negative error code on error. + */ +int xe_gt_tlb_invalidation_guc(struct xe_gt *gt) +{ + u32 action[] = { + XE_GUC_ACTION_TLB_INVALIDATION, + 0, /* seqno, replaced in send_tlb_invalidation */ + MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC), + }; + + return send_tlb_invalidation(>->uc.guc, NULL, action, + ARRAY_SIZE(action)); +} + +/** + * xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA + * @gt: graphics tile + * @fence: invalidation fence which will be signal on TLB invalidation + * completion, can be NULL + * @vma: VMA to invalidate + * + * Issue a range based TLB invalidation if supported, if not fallback to a full + * TLB invalidation. Completion of TLB is asynchronous and caller can either use + * the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for + * completion. + * + * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success, + * negative error code on error. + */ +int xe_gt_tlb_invalidation_vma(struct xe_gt *gt, + struct xe_gt_tlb_invalidation_fence *fence, + struct xe_vma *vma) +{ + struct xe_device *xe = gt_to_xe(gt); +#define MAX_TLB_INVALIDATION_LEN 7 + u32 action[MAX_TLB_INVALIDATION_LEN]; + int len = 0; + + xe_gt_assert(gt, vma); + + action[len++] = XE_GUC_ACTION_TLB_INVALIDATION; + action[len++] = 0; /* seqno, replaced in send_tlb_invalidation */ + if (!xe->info.has_range_tlb_invalidation) { + action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL); + } else { + u64 start = xe_vma_start(vma); + u64 length = xe_vma_size(vma); + u64 align, end; + + if (length < SZ_4K) + length = SZ_4K; + + /* + * We need to invalidate a higher granularity if start address + * is not aligned to length. When start is not aligned with + * length we need to find the length large enough to create an + * address mask covering the required range. + */ + align = roundup_pow_of_two(length); + start = ALIGN_DOWN(xe_vma_start(vma), align); + end = ALIGN(xe_vma_end(vma), align); + length = align; + while (start + length < end) { + length <<= 1; + start = ALIGN_DOWN(xe_vma_start(vma), length); + } + + /* + * Minimum invalidation size for a 2MB page that the hardware + * expects is 16MB + */ + if (length >= SZ_2M) { + length = max_t(u64, SZ_16M, length); + start = ALIGN_DOWN(xe_vma_start(vma), length); + } + + xe_gt_assert(gt, length >= SZ_4K); + xe_gt_assert(gt, is_power_of_2(length)); + xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1, ilog2(SZ_2M) + 1))); + xe_gt_assert(gt, IS_ALIGNED(start, length)); + + action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE); + action[len++] = xe_vma_vm(vma)->usm.asid; + action[len++] = lower_32_bits(start); + action[len++] = upper_32_bits(start); + action[len++] = ilog2(length) - ilog2(SZ_4K); + } + + xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN); + + return send_tlb_invalidation(>->uc.guc, fence, action, len); +} + +/** + * xe_gt_tlb_invalidation_wait - Wait for TLB to complete + * @gt: graphics tile + * @seqno: seqno to wait which was returned from xe_gt_tlb_invalidation + * + * Wait for 200ms for a TLB invalidation to complete, in practice we always + * should receive the TLB invalidation within 200ms. + * + * Return: 0 on success, -ETIME on TLB invalidation timeout + */ +int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_guc *guc = >->uc.guc; + struct drm_printer p = drm_err_printer(__func__); + int ret; + + /* + * XXX: See above, this algorithm only works if seqno are always in + * order + */ + ret = wait_event_timeout(guc->ct.wq, + tlb_invalidation_seqno_past(gt, seqno), + TLB_TIMEOUT); + if (!ret) { + drm_err(&xe->drm, "gt%d: TLB invalidation time'd out, seqno=%d, recv=%d\n", + gt->info.id, seqno, gt->tlb_invalidation.seqno_recv); + xe_guc_ct_print(&guc->ct, &p, true); + return -ETIME; + } + + return 0; +} + +/** + * xe_guc_tlb_invalidation_done_handler - TLB invalidation done handler + * @guc: guc + * @msg: message indicating TLB invalidation done + * @len: length of message + * + * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any + * invalidation fences for seqno. Algorithm for this depends on seqno being + * received in-order and asserts this assumption. + * + * Return: 0 on success, -EPROTO for malformed messages. + */ +int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_gt_tlb_invalidation_fence *fence, *next; + unsigned long flags; + + if (unlikely(len != 1)) + return -EPROTO; + + /* + * This can also be run both directly from the IRQ handler and also in + * process_g2h_msg(). Only one may process any individual CT message, + * however the order they are processed here could result in skipping a + * seqno. To handle that we just process all the seqnos from the last + * seqno_recv up to and including the one in msg[0]. The delta should be + * very small so there shouldn't be much of pending_fences we actually + * need to iterate over here. + * + * From GuC POV we expect the seqnos to always appear in-order, so if we + * see something later in the timeline we can be sure that anything + * appearing earlier has already signalled, just that we have yet to + * officially process the CT message like if racing against + * process_g2h_msg(). + */ + spin_lock_irqsave(>->tlb_invalidation.pending_lock, flags); + if (tlb_invalidation_seqno_past(gt, msg[0])) { + spin_unlock_irqrestore(>->tlb_invalidation.pending_lock, flags); + return 0; + } + + /* + * wake_up_all() and wait_event_timeout() already have the correct + * barriers. + */ + WRITE_ONCE(gt->tlb_invalidation.seqno_recv, msg[0]); + wake_up_all(&guc->ct.wq); + + list_for_each_entry_safe(fence, next, + >->tlb_invalidation.pending_fences, link) { + trace_xe_gt_tlb_invalidation_fence_recv(fence); + + if (!tlb_invalidation_seqno_past(gt, fence->seqno)) + break; + + invalidation_fence_signal(fence); + } + + if (!list_empty(>->tlb_invalidation.pending_fences)) + mod_delayed_work(system_wq, + >->tlb_invalidation.fence_tdr, + TLB_TIMEOUT); + else + cancel_delayed_work(>->tlb_invalidation.fence_tdr); + + spin_unlock_irqrestore(>->tlb_invalidation.pending_lock, flags); + + return 0; +} |