diff options
Diffstat (limited to 'drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c')
| -rw-r--r-- | drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c | 645 |
1 files changed, 425 insertions, 220 deletions
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 1ce7e04aa837..61a6f2424e24 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -25,6 +25,7 @@ #include "gt/intel_ring.h" #include "intel_guc_ads.h" +#include "intel_guc_capture.h" #include "intel_guc_submission.h" #include "i915_drv.h" @@ -161,7 +162,8 @@ guc_create_parallel(struct intel_engine_cs **engines, #define SCHED_STATE_ENABLED BIT(4) #define SCHED_STATE_PENDING_ENABLE BIT(5) #define SCHED_STATE_REGISTERED BIT(6) -#define SCHED_STATE_BLOCKED_SHIFT 7 +#define SCHED_STATE_POLICY_REQUIRED BIT(7) +#define SCHED_STATE_BLOCKED_SHIFT 8 #define SCHED_STATE_BLOCKED BIT(SCHED_STATE_BLOCKED_SHIFT) #define SCHED_STATE_BLOCKED_MASK (0xfff << SCHED_STATE_BLOCKED_SHIFT) @@ -300,6 +302,23 @@ static inline void clr_context_registered(struct intel_context *ce) ce->guc_state.sched_state &= ~SCHED_STATE_REGISTERED; } +static inline bool context_policy_required(struct intel_context *ce) +{ + return ce->guc_state.sched_state & SCHED_STATE_POLICY_REQUIRED; +} + +static inline void set_context_policy_required(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state |= SCHED_STATE_POLICY_REQUIRED; +} + +static inline void clr_context_policy_required(struct intel_context *ce) +{ + lockdep_assert_held(&ce->guc_state.lock); + ce->guc_state.sched_state &= ~SCHED_STATE_POLICY_REQUIRED; +} + static inline u32 context_blocked(struct intel_context *ce) { return (ce->guc_state.sched_state & SCHED_STATE_BLOCKED_MASK) >> @@ -351,12 +370,12 @@ request_to_scheduling_context(struct i915_request *rq) static inline bool context_guc_id_invalid(struct intel_context *ce) { - return ce->guc_id.id == GUC_INVALID_LRC_ID; + return ce->guc_id.id == GUC_INVALID_CONTEXT_ID; } static inline void set_context_guc_id_invalid(struct intel_context *ce) { - ce->guc_id.id = GUC_INVALID_LRC_ID; + ce->guc_id.id = GUC_INVALID_CONTEXT_ID; } static inline struct intel_guc *ce_to_guc(struct intel_context *ce) @@ -395,12 +414,12 @@ struct sync_semaphore { }; struct parent_scratch { - struct guc_process_desc pdesc; + struct guc_sched_wq_desc wq_desc; struct sync_semaphore go; struct sync_semaphore join[MAX_ENGINE_INSTANCE + 1]; - u8 unused[WQ_OFFSET - sizeof(struct guc_process_desc) - + u8 unused[WQ_OFFSET - sizeof(struct guc_sched_wq_desc) - sizeof(struct sync_semaphore) * (MAX_ENGINE_INSTANCE + 2)]; u32 wq[WQ_SIZE / sizeof(u32)]; @@ -437,15 +456,15 @@ __get_parent_scratch(struct intel_context *ce) LRC_STATE_OFFSET) / sizeof(u32))); } -static struct guc_process_desc * -__get_process_desc(struct intel_context *ce) +static struct guc_sched_wq_desc * +__get_wq_desc(struct intel_context *ce) { struct parent_scratch *ps = __get_parent_scratch(ce); - return &ps->pdesc; + return &ps->wq_desc; } -static u32 *get_wq_pointer(struct guc_process_desc *desc, +static u32 *get_wq_pointer(struct guc_sched_wq_desc *wq_desc, struct intel_context *ce, u32 wqi_size) { @@ -457,7 +476,7 @@ static u32 *get_wq_pointer(struct guc_process_desc *desc, #define AVAILABLE_SPACE \ CIRC_SPACE(ce->parallel.guc.wqi_tail, ce->parallel.guc.wqi_head, WQ_SIZE) if (wqi_size > AVAILABLE_SPACE) { - ce->parallel.guc.wqi_head = READ_ONCE(desc->head); + ce->parallel.guc.wqi_head = READ_ONCE(wq_desc->head); if (wqi_size > AVAILABLE_SPACE) return NULL; @@ -467,75 +486,27 @@ static u32 *get_wq_pointer(struct guc_process_desc *desc, return &__get_parent_scratch(ce)->wq[ce->parallel.guc.wqi_tail / sizeof(u32)]; } -static struct guc_lrc_desc *__get_lrc_desc(struct intel_guc *guc, u32 index) -{ - struct guc_lrc_desc *base = guc->lrc_desc_pool_vaddr; - - GEM_BUG_ON(index >= GUC_MAX_LRC_DESCRIPTORS); - - return &base[index]; -} - static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id) { struct intel_context *ce = xa_load(&guc->context_lookup, id); - GEM_BUG_ON(id >= GUC_MAX_LRC_DESCRIPTORS); + GEM_BUG_ON(id >= GUC_MAX_CONTEXT_ID); return ce; } -static int guc_lrc_desc_pool_create(struct intel_guc *guc) -{ - u32 size; - int ret; - - size = PAGE_ALIGN(sizeof(struct guc_lrc_desc) * - GUC_MAX_LRC_DESCRIPTORS); - ret = intel_guc_allocate_and_map_vma(guc, size, &guc->lrc_desc_pool, - (void **)&guc->lrc_desc_pool_vaddr); - if (ret) - return ret; - - return 0; -} - -static void guc_lrc_desc_pool_destroy(struct intel_guc *guc) -{ - guc->lrc_desc_pool_vaddr = NULL; - i915_vma_unpin_and_release(&guc->lrc_desc_pool, I915_VMA_RELEASE_MAP); -} - static inline bool guc_submission_initialized(struct intel_guc *guc) { - return !!guc->lrc_desc_pool_vaddr; + return guc->submission_initialized; } -static inline void reset_lrc_desc(struct intel_guc *guc, u32 id) -{ - if (likely(guc_submission_initialized(guc))) { - struct guc_lrc_desc *desc = __get_lrc_desc(guc, id); - unsigned long flags; - - memset(desc, 0, sizeof(*desc)); - - /* - * xarray API doesn't have xa_erase_irqsave wrapper, so calling - * the lower level functions directly. - */ - xa_lock_irqsave(&guc->context_lookup, flags); - __xa_erase(&guc->context_lookup, id); - xa_unlock_irqrestore(&guc->context_lookup, flags); - } -} - -static inline bool lrc_desc_registered(struct intel_guc *guc, u32 id) +static inline bool ctx_id_mapped(struct intel_guc *guc, u32 id) { return __get_context(guc, id); } -static inline void set_lrc_desc_registered(struct intel_guc *guc, u32 id, - struct intel_context *ce) +static inline void set_ctx_id_mapping(struct intel_guc *guc, u32 id, + struct intel_context *ce) { unsigned long flags; @@ -548,6 +519,22 @@ static inline void set_lrc_desc_registered(struct intel_guc *guc, u32 id, xa_unlock_irqrestore(&guc->context_lookup, flags); } +static inline void clr_ctx_id_mapping(struct intel_guc *guc, u32 id) +{ + unsigned long flags; + + if (unlikely(!guc_submission_initialized(guc))) + return; + + /* + * xarray API doesn't have xa_erase_irqsave wrapper, so calling + * the lower level functions directly. + */ + xa_lock_irqsave(&guc->context_lookup, flags); + __xa_erase(&guc->context_lookup, id); + xa_unlock_irqrestore(&guc->context_lookup, flags); +} + static void decr_outstanding_submission_g2h(struct intel_guc *guc) { if (atomic_dec_and_test(&guc->outstanding_submission_g2h)) @@ -624,7 +611,8 @@ int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout) true, timeout); } -static int guc_lrc_desc_pin(struct intel_context *ce, bool loop); +static int guc_context_policy_init(struct intel_context *ce, bool loop); +static int try_context_registration(struct intel_context *ce, bool loop); static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq) { @@ -650,6 +638,12 @@ static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq) GEM_BUG_ON(!atomic_read(&ce->guc_id.ref)); GEM_BUG_ON(context_guc_id_invalid(ce)); + if (context_policy_required(ce)) { + err = guc_context_policy_init(ce, false); + if (err) + return err; + } + spin_lock(&ce->guc_state.lock); /* @@ -743,7 +737,7 @@ static u32 wq_space_until_wrap(struct intel_context *ce) return (WQ_SIZE - ce->parallel.guc.wqi_tail); } -static void write_wqi(struct guc_process_desc *desc, +static void write_wqi(struct guc_sched_wq_desc *wq_desc, struct intel_context *ce, u32 wqi_size) { @@ -756,13 +750,13 @@ static void write_wqi(struct guc_process_desc *desc, ce->parallel.guc.wqi_tail = (ce->parallel.guc.wqi_tail + wqi_size) & (WQ_SIZE - 1); - WRITE_ONCE(desc->tail, ce->parallel.guc.wqi_tail); + WRITE_ONCE(wq_desc->tail, ce->parallel.guc.wqi_tail); } static int guc_wq_noop_append(struct intel_context *ce) { - struct guc_process_desc *desc = __get_process_desc(ce); - u32 *wqi = get_wq_pointer(desc, ce, wq_space_until_wrap(ce)); + struct guc_sched_wq_desc *wq_desc = __get_wq_desc(ce); + u32 *wqi = get_wq_pointer(wq_desc, ce, wq_space_until_wrap(ce)); u32 len_dw = wq_space_until_wrap(ce) / sizeof(u32) - 1; if (!wqi) @@ -781,7 +775,7 @@ static int __guc_wq_item_append(struct i915_request *rq) { struct intel_context *ce = request_to_scheduling_context(rq); struct intel_context *child; - struct guc_process_desc *desc = __get_process_desc(ce); + struct guc_sched_wq_desc *wq_desc = __get_wq_desc(ce); unsigned int wqi_size = (ce->parallel.number_children + 4) * sizeof(u32); u32 *wqi; @@ -792,7 +786,7 @@ static int __guc_wq_item_append(struct i915_request *rq) GEM_BUG_ON(!atomic_read(&ce->guc_id.ref)); GEM_BUG_ON(context_guc_id_invalid(ce)); GEM_BUG_ON(context_wait_for_deregister_to_register(ce)); - GEM_BUG_ON(!lrc_desc_registered(ce_to_guc(ce), ce->guc_id.id)); + GEM_BUG_ON(!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id)); /* Insert NOOP if this work queue item will wrap the tail pointer. */ if (wqi_size > wq_space_until_wrap(ce)) { @@ -801,7 +795,7 @@ static int __guc_wq_item_append(struct i915_request *rq) return ret; } - wqi = get_wq_pointer(desc, ce, wqi_size); + wqi = get_wq_pointer(wq_desc, ce, wqi_size); if (!wqi) return -EBUSY; @@ -816,7 +810,7 @@ static int __guc_wq_item_append(struct i915_request *rq) for_each_child(ce, child) *wqi++ = child->ring->tail / sizeof(u64); - write_wqi(desc, ce, wqi_size); + write_wqi(wq_desc, ce, wqi_size); return 0; } @@ -920,9 +914,9 @@ register_context: if (submit) { struct intel_context *ce = request_to_scheduling_context(last); - if (unlikely(!lrc_desc_registered(guc, ce->guc_id.id) && + if (unlikely(!ctx_id_mapped(guc, ce->guc_id.id) && !intel_context_is_banned(ce))) { - ret = guc_lrc_desc_pin(ce, false); + ret = try_context_registration(ce, false); if (unlikely(ret == -EPIPE)) { goto deadlk; } else if (ret == -EBUSY) { @@ -1546,6 +1540,89 @@ static void guc_reset_state(struct intel_context *ce, u32 head, bool scrub) lrc_update_regs(ce, engine, head); } +static u32 __cs_pending_mi_force_wakes(struct intel_engine_cs *engine) +{ + static const i915_reg_t _reg[I915_NUM_ENGINES] = { + [RCS0] = MSG_IDLE_CS, + [BCS0] = MSG_IDLE_BCS, + [VCS0] = MSG_IDLE_VCS0, + [VCS1] = MSG_IDLE_VCS1, + [VCS2] = MSG_IDLE_VCS2, + [VCS3] = MSG_IDLE_VCS3, + [VCS4] = MSG_IDLE_VCS4, + [VCS5] = MSG_IDLE_VCS5, + [VCS6] = MSG_IDLE_VCS6, + [VCS7] = MSG_IDLE_VCS7, + [VECS0] = MSG_IDLE_VECS0, + [VECS1] = MSG_IDLE_VECS1, + [VECS2] = MSG_IDLE_VECS2, + [VECS3] = MSG_IDLE_VECS3, + [CCS0] = MSG_IDLE_CS, + [CCS1] = MSG_IDLE_CS, + [CCS2] = MSG_IDLE_CS, + [CCS3] = MSG_IDLE_CS, + }; + u32 val; + + if (!_reg[engine->id].reg) + return 0; + + val = intel_uncore_read(engine->uncore, _reg[engine->id]); + + /* bits[29:25] & bits[13:9] >> shift */ + return (val & (val >> 16) & MSG_IDLE_FW_MASK) >> MSG_IDLE_FW_SHIFT; +} + +static void __gpm_wait_for_fw_complete(struct intel_gt *gt, u32 fw_mask) +{ + int ret; + + /* Ensure GPM receives fw up/down after CS is stopped */ + udelay(1); + + /* Wait for forcewake request to complete in GPM */ + ret = __intel_wait_for_register_fw(gt->uncore, + GEN9_PWRGT_DOMAIN_STATUS, + fw_mask, fw_mask, 5000, 0, NULL); + + /* Ensure CS receives fw ack from GPM */ + udelay(1); + + if (ret) + GT_TRACE(gt, "Failed to complete pending forcewake %d\n", ret); +} + +/* + * Wa_22011802037:gen12: In addition to stopping the cs, we need to wait for any + * pending MI_FORCE_WAKEUP requests that the CS has initiated to complete. The + * pending status is indicated by bits[13:9] (masked by bits[ 29:25]) in the + * MSG_IDLE register. There's one MSG_IDLE register per reset domain. Since we + * are concerned only with the gt reset here, we use a logical OR of pending + * forcewakeups from all reset domains and then wait for them to complete by + * querying PWRGT_DOMAIN_STATUS. + */ +static void guc_engine_reset_prepare(struct intel_engine_cs *engine) +{ + u32 fw_pending; + + if (GRAPHICS_VER(engine->i915) != 12) + return; + + /* + * Wa_22011802037 + * TODO: Occasionally trying to stop the cs times out, but does not + * adversely affect functionality. The timeout is set as a config + * parameter that defaults to 100ms. Assuming that this timeout is + * sufficient for any pending MI_FORCEWAKEs to complete, ignore the + * timeout returned here until it is root caused. + */ + intel_engine_stop_cs(engine); + + fw_pending = __cs_pending_mi_force_wakes(engine); + if (fw_pending) + __gpm_wait_for_fw_complete(engine->gt, fw_pending); +} + static void guc_reset_nop(struct intel_engine_cs *engine) { } @@ -1804,20 +1881,10 @@ static void reset_fail_worker_func(struct work_struct *w); int intel_guc_submission_init(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); - int ret; - if (guc->lrc_desc_pool) + if (guc->submission_initialized) return 0; - ret = guc_lrc_desc_pool_create(guc); - if (ret) - return ret; - /* - * Keep static analysers happy, let them know that we allocated the - * vma after testing that it didn't exist earlier. - */ - GEM_BUG_ON(!guc->lrc_desc_pool); - guc->submission_state.guc_ids_bitmap = bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc), GFP_KERNEL); if (!guc->submission_state.guc_ids_bitmap) @@ -1825,19 +1892,20 @@ int intel_guc_submission_init(struct intel_guc *guc) guc->timestamp.ping_delay = (POLL_TIME_CLKS / gt->clock_frequency + 1) * HZ; guc->timestamp.shift = gpm_timestamp_shift(gt); + guc->submission_initialized = true; return 0; } void intel_guc_submission_fini(struct intel_guc *guc) { - if (!guc->lrc_desc_pool) + if (!guc->submission_initialized) return; guc_flush_destroyed_contexts(guc); - guc_lrc_desc_pool_destroy(guc); i915_sched_engine_put(guc->sched_engine); bitmap_free(guc->submission_state.guc_ids_bitmap); + guc->submission_initialized = false; } static inline void queue_request(struct i915_sched_engine *sched_engine, @@ -1884,7 +1952,7 @@ static bool need_tasklet(struct intel_guc *guc, struct i915_request *rq) return submission_disabled(guc) || guc->stalled_request || !i915_sched_engine_is_empty(sched_engine) || - !lrc_desc_registered(guc, ce->guc_id.id); + !ctx_id_mapped(guc, ce->guc_id.id); } static void guc_submit_request(struct i915_request *rq) @@ -1941,7 +2009,7 @@ static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce) else ida_simple_remove(&guc->submission_state.guc_ids, ce->guc_id.id); - reset_lrc_desc(guc, ce->guc_id.id); + clr_ctx_id_mapping(guc, ce->guc_id.id); set_context_guc_id_invalid(ce); } if (!list_empty(&ce->guc_id.link)) @@ -2094,65 +2162,96 @@ static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce) static int __guc_action_register_multi_lrc(struct intel_guc *guc, struct intel_context *ce, - u32 guc_id, - u32 offset, + struct guc_ctxt_registration_info *info, bool loop) { struct intel_context *child; - u32 action[4 + MAX_ENGINE_INSTANCE]; + u32 action[13 + (MAX_ENGINE_INSTANCE * 2)]; int len = 0; + u32 next_id; GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE); action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; - action[len++] = guc_id; + action[len++] = info->flags; + action[len++] = info->context_idx; + action[len++] = info->engine_class; + action[len++] = info->engine_submit_mask; + action[len++] = info->wq_desc_lo; + action[len++] = info->wq_desc_hi; + action[len++] = info->wq_base_lo; + action[len++] = info->wq_base_hi; + action[len++] = info->wq_size; action[len++] = ce->parallel.number_children + 1; - action[len++] = offset; + action[len++] = info->hwlrca_lo; + action[len++] = info->hwlrca_hi; + + next_id = info->context_idx + 1; for_each_child(ce, child) { - offset += sizeof(struct guc_lrc_desc); - action[len++] = offset; + GEM_BUG_ON(next_id++ != child->guc_id.id); + + /* + * NB: GuC interface supports 64 bit LRCA even though i915/HW + * only supports 32 bit currently. + */ + action[len++] = lower_32_bits(child->lrc.lrca); + action[len++] = upper_32_bits(child->lrc.lrca); } + GEM_BUG_ON(len > ARRAY_SIZE(action)); + return guc_submission_send_busy_loop(guc, action, len, 0, loop); } static int __guc_action_register_context(struct intel_guc *guc, - u32 guc_id, - u32 offset, + struct guc_ctxt_registration_info *info, bool loop) { u32 action[] = { INTEL_GUC_ACTION_REGISTER_CONTEXT, - guc_id, - offset, + info->flags, + info->context_idx, + info->engine_class, + info->engine_submit_mask, + info->wq_desc_lo, + info->wq_desc_hi, + info->wq_base_lo, + info->wq_base_hi, + info->wq_size, + info->hwlrca_lo, + info->hwlrca_hi, }; return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, loop); } +static void prepare_context_registration_info(struct intel_context *ce, + struct guc_ctxt_registration_info *info); + static int register_context(struct intel_context *ce, bool loop) { + struct guc_ctxt_registration_info info; struct intel_guc *guc = ce_to_guc(ce); - u32 offset = intel_guc_ggtt_offset(guc, guc->lrc_desc_pool) + - ce->guc_id.id * sizeof(struct guc_lrc_desc); int ret; GEM_BUG_ON(intel_context_is_child(ce)); trace_intel_context_register(ce); + prepare_context_registration_info(ce, &info); + if (intel_context_is_parent(ce)) - ret = __guc_action_register_multi_lrc(guc, ce, ce->guc_id.id, - offset, loop); + ret = __guc_action_register_multi_lrc(guc, ce, &info, loop); else - ret = __guc_action_register_context(guc, ce->guc_id.id, offset, - loop); + ret = __guc_action_register_context(guc, &info, loop); if (likely(!ret)) { unsigned long flags; spin_lock_irqsave(&ce->guc_state.lock, flags); set_context_registered(ce); spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + guc_context_policy_init(ce, loop); } return ret; @@ -2202,33 +2301,120 @@ static inline u32 get_children_join_value(struct intel_context *ce, return __get_parent_scratch(ce)->join[child_index].semaphore; } -static void guc_context_policy_init(struct intel_engine_cs *engine, - struct guc_lrc_desc *desc) +struct context_policy { + u32 count; + struct guc_update_context_policy h2g; +}; + +static u32 __guc_context_policy_action_size(struct context_policy *policy) { - desc->policy_flags = 0; + size_t bytes = sizeof(policy->h2g.header) + + (sizeof(policy->h2g.klv[0]) * policy->count); - if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION) - desc->policy_flags |= CONTEXT_POLICY_FLAG_PREEMPT_TO_IDLE; + return bytes / sizeof(u32); +} + +static void __guc_context_policy_start_klv(struct context_policy *policy, u16 guc_id) +{ + policy->h2g.header.action = INTEL_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; + policy->h2g.header.ctx_id = guc_id; + policy->count = 0; +} + +#define MAKE_CONTEXT_POLICY_ADD(func, id) \ +static void __guc_context_policy_add_##func(struct context_policy *policy, u32 data) \ +{ \ + GEM_BUG_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ + policy->h2g.klv[policy->count].kl = \ + FIELD_PREP(GUC_KLV_0_KEY, GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ + FIELD_PREP(GUC_KLV_0_LEN, 1); \ + policy->h2g.klv[policy->count].value = data; \ + policy->count++; \ +} + +MAKE_CONTEXT_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) +MAKE_CONTEXT_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) +MAKE_CONTEXT_POLICY_ADD(priority, SCHEDULING_PRIORITY) +MAKE_CONTEXT_POLICY_ADD(preempt_to_idle, PREEMPT_TO_IDLE_ON_QUANTUM_EXPIRY) + +#undef MAKE_CONTEXT_POLICY_ADD + +static int __guc_context_set_context_policies(struct intel_guc *guc, + struct context_policy *policy, + bool loop) +{ + return guc_submission_send_busy_loop(guc, (u32 *)&policy->h2g, + __guc_context_policy_action_size(policy), + 0, loop); +} + +static int guc_context_policy_init(struct intel_context *ce, bool loop) +{ + struct intel_engine_cs *engine = ce->engine; + struct intel_guc *guc = &engine->gt->uc.guc; + struct context_policy policy; + u32 execution_quantum; + u32 preemption_timeout; + bool missing = false; + unsigned long flags; + int ret; /* NB: For both of these, zero means disabled. */ - desc->execution_quantum = engine->props.timeslice_duration_ms * 1000; - desc->preemption_timeout = engine->props.preempt_timeout_ms * 1000; + execution_quantum = engine->props.timeslice_duration_ms * 1000; + preemption_timeout = engine->props.preempt_timeout_ms * 1000; + + __guc_context_policy_start_klv(&policy, ce->guc_id.id); + + __guc_context_policy_add_priority(&policy, ce->guc_state.prio); + __guc_context_policy_add_execution_quantum(&policy, execution_quantum); + __guc_context_policy_add_preemption_timeout(&policy, preemption_timeout); + + if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION) + __guc_context_policy_add_preempt_to_idle(&policy, 1); + + ret = __guc_context_set_context_policies(guc, &policy, loop); + missing = ret != 0; + + if (!missing && intel_context_is_parent(ce)) { + struct intel_context *child; + + for_each_child(ce, child) { + __guc_context_policy_start_klv(&policy, child->guc_id.id); + + if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION) + __guc_context_policy_add_preempt_to_idle(&policy, 1); + + child->guc_state.prio = ce->guc_state.prio; + __guc_context_policy_add_priority(&policy, ce->guc_state.prio); + __guc_context_policy_add_execution_quantum(&policy, execution_quantum); + __guc_context_policy_add_preemption_timeout(&policy, preemption_timeout); + + ret = __guc_context_set_context_policies(guc, &policy, loop); + if (ret) { + missing = true; + break; + } + } + } + + spin_lock_irqsave(&ce->guc_state.lock, flags); + if (missing) + set_context_policy_required(ce); + else + clr_context_policy_required(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + return ret; } -static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) +static void prepare_context_registration_info(struct intel_context *ce, + struct guc_ctxt_registration_info *info) { struct intel_engine_cs *engine = ce->engine; - struct intel_runtime_pm *runtime_pm = engine->uncore->rpm; struct intel_guc *guc = &engine->gt->uc.guc; - u32 desc_idx = ce->guc_id.id; - struct guc_lrc_desc *desc; - bool context_registered; - intel_wakeref_t wakeref; - struct intel_context *child; - int ret = 0; + u32 ctx_id = ce->guc_id.id; GEM_BUG_ON(!engine->mask); - GEM_BUG_ON(!sched_state_is_init(ce)); /* * Ensure LRC + CT vmas are is same region as write barrier is done @@ -2237,55 +2423,63 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) != i915_gem_object_is_lmem(ce->ring->vma->obj)); - context_registered = lrc_desc_registered(guc, desc_idx); - - reset_lrc_desc(guc, desc_idx); - set_lrc_desc_registered(guc, desc_idx, ce); - - desc = __get_lrc_desc(guc, desc_idx); - desc->engine_class = engine_class_to_guc_class(engine->class); - desc->engine_submit_mask = engine->logical_mask; - desc->hw_context_desc = ce->lrc.lrca; - desc->priority = ce->guc_state.prio; - desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD; - guc_context_policy_init(engine, desc); + memset(info, 0, sizeof(*info)); + info->context_idx = ctx_id; + info->engine_class = engine_class_to_guc_class(engine->class); + info->engine_submit_mask = engine->logical_mask; + /* + * NB: GuC interface supports 64 bit LRCA even though i915/HW + * only supports 32 bit currently. + */ + info->hwlrca_lo = lower_32_bits(ce->lrc.lrca); + info->hwlrca_hi = upper_32_bits(ce->lrc.lrca); + info->flags = CONTEXT_REGISTRATION_FLAG_KMD; /* * If context is a parent, we need to register a process descriptor * describing a work queue and register all child contexts. */ if (intel_context_is_parent(ce)) { - struct guc_process_desc *pdesc; + struct guc_sched_wq_desc *wq_desc; + u64 wq_desc_offset, wq_base_offset; ce->parallel.guc.wqi_tail = 0; ce->parallel.guc.wqi_head = 0; - desc->process_desc = i915_ggtt_offset(ce->state) + - __get_parent_scratch_offset(ce); - desc->wq_addr = i915_ggtt_offset(ce->state) + - __get_wq_offset(ce); - desc->wq_size = WQ_SIZE; + wq_desc_offset = i915_ggtt_offset(ce->state) + + __get_parent_scratch_offset(ce); + wq_base_offset = i915_ggtt_offset(ce->state) + + __get_wq_offset(ce); + info->wq_desc_lo = lower_32_bits(wq_desc_offset); + info->wq_desc_hi = upper_32_bits(wq_desc_offset); + info->wq_base_lo = lower_32_bits(wq_base_offset); + info->wq_base_hi = upper_32_bits(wq_base_offset); + info->wq_size = WQ_SIZE; - pdesc = __get_process_desc(ce); - memset(pdesc, 0, sizeof(*(pdesc))); - pdesc->stage_id = ce->guc_id.id; - pdesc->wq_base_addr = desc->wq_addr; - pdesc->wq_size_bytes = desc->wq_size; - pdesc->wq_status = WQ_STATUS_ACTIVE; - - for_each_child(ce, child) { - desc = __get_lrc_desc(guc, child->guc_id.id); - - desc->engine_class = - engine_class_to_guc_class(engine->class); - desc->hw_context_desc = child->lrc.lrca; - desc->priority = ce->guc_state.prio; - desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD; - guc_context_policy_init(engine, desc); - } + wq_desc = __get_wq_desc(ce); + memset(wq_desc, 0, sizeof(*wq_desc)); + wq_desc->wq_status = WQ_STATUS_ACTIVE; clear_children_join_go_memory(ce); } +} + +static int try_context_registration(struct intel_context *ce, bool loop) +{ + struct intel_engine_cs *engine = ce->engine; + struct intel_runtime_pm *runtime_pm = engine->uncore->rpm; + struct intel_guc *guc = &engine->gt->uc.guc; + intel_wakeref_t wakeref; + u32 ctx_id = ce->guc_id.id; + bool context_registered; + int ret = 0; + + GEM_BUG_ON(!sched_state_is_init(ce)); + + context_registered = ctx_id_mapped(guc, ctx_id); + + clr_ctx_id_mapping(guc, ctx_id); + set_ctx_id_mapping(guc, ctx_id, ce); /* * The context_lookup xarray is used to determine if the hardware @@ -2311,7 +2505,7 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) } spin_unlock_irqrestore(&ce->guc_state.lock, flags); if (unlikely(disabled)) { - reset_lrc_desc(guc, desc_idx); + clr_ctx_id_mapping(guc, ctx_id); return 0; /* Will get registered later */ } @@ -2327,9 +2521,9 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) with_intel_runtime_pm(runtime_pm, wakeref) ret = register_context(ce, loop); if (unlikely(ret == -EBUSY)) { - reset_lrc_desc(guc, desc_idx); + clr_ctx_id_mapping(guc, ctx_id); } else if (unlikely(ret == -ENODEV)) { - reset_lrc_desc(guc, desc_idx); + clr_ctx_id_mapping(guc, ctx_id); ret = 0; /* Will get registered later */ } } @@ -2419,7 +2613,7 @@ static void __guc_context_sched_disable(struct intel_guc *guc, GUC_CONTEXT_DISABLE }; - GEM_BUG_ON(guc_id == GUC_INVALID_LRC_ID); + GEM_BUG_ON(guc_id == GUC_INVALID_CONTEXT_ID); GEM_BUG_ON(intel_context_is_child(ce)); trace_intel_context_sched_disable(ce); @@ -2516,7 +2710,7 @@ static bool context_cant_unblock(struct intel_context *ce) return (ce->guc_state.sched_state & SCHED_STATE_NO_UNBLOCK) || context_guc_id_invalid(ce) || - !lrc_desc_registered(ce_to_guc(ce), ce->guc_id.id) || + !ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id) || !intel_context_is_pinned(ce); } @@ -2580,13 +2774,11 @@ static void __guc_context_set_preemption_timeout(struct intel_guc *guc, u16 guc_id, u32 preemption_timeout) { - u32 action[] = { - INTEL_GUC_ACTION_SET_CONTEXT_PREEMPTION_TIMEOUT, - guc_id, - preemption_timeout - }; + struct context_policy policy; - intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); + __guc_context_policy_start_klv(&policy, guc_id); + __guc_context_policy_add_preemption_timeout(&policy, preemption_timeout); + __guc_context_set_context_policies(guc, &policy, true); } static void guc_context_ban(struct intel_context *ce, struct i915_request *rq) @@ -2686,7 +2878,7 @@ static inline void guc_lrc_desc_unpin(struct intel_context *ce) bool disabled; GEM_BUG_ON(!intel_gt_pm_is_awake(gt)); - GEM_BUG_ON(!lrc_desc_registered(guc, ce->guc_id.id)); + GEM_BUG_ON(!ctx_id_mapped(guc, ce->guc_id.id)); GEM_BUG_ON(ce != __get_context(guc, ce->guc_id.id)); GEM_BUG_ON(context_enabled(ce)); @@ -2803,7 +2995,7 @@ static void guc_context_destroy(struct kref *kref) */ spin_lock_irqsave(&guc->submission_state.lock, flags); destroy = submission_disabled(guc) || context_guc_id_invalid(ce) || - !lrc_desc_registered(guc, ce->guc_id.id); + !ctx_id_mapped(guc, ce->guc_id.id); if (likely(!destroy)) { if (!list_empty(&ce->guc_id.link)) list_del_init(&ce->guc_id.link); @@ -2831,16 +3023,20 @@ static int guc_context_alloc(struct intel_context *ce) return lrc_alloc(ce, ce->engine); } +static void __guc_context_set_prio(struct intel_guc *guc, + struct intel_context *ce) +{ + struct context_policy policy; + + __guc_context_policy_start_klv(&policy, ce->guc_id.id); + __guc_context_policy_add_priority(&policy, ce->guc_state.prio); + __guc_context_set_context_policies(guc, &policy, true); +} + static void guc_context_set_prio(struct intel_guc *guc, struct intel_context *ce, u8 prio) { - u32 action[] = { - INTEL_GUC_ACTION_SET_CONTEXT_PRIORITY, - ce->guc_id.id, - prio, - }; - GEM_BUG_ON(prio < GUC_CLIENT_PRIORITY_KMD_HIGH || prio > GUC_CLIENT_PRIORITY_NORMAL); lockdep_assert_held(&ce->guc_state.lock); @@ -2851,9 +3047,9 @@ static void guc_context_set_prio(struct intel_guc *guc, return; } - guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); - ce->guc_state.prio = prio; + __guc_context_set_prio(guc, ce); + trace_intel_context_set_prio(ce); } @@ -3046,7 +3242,7 @@ static void guc_signal_context_fence(struct intel_context *ce) static bool context_needs_register(struct intel_context *ce, bool new_guc_id) { return (new_guc_id || test_bit(CONTEXT_LRCA_DIRTY, &ce->flags) || - !lrc_desc_registered(ce_to_guc(ce), ce->guc_id.id)) && + !ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id)) && !submission_disabled(ce_to_guc(ce)); } @@ -3123,7 +3319,7 @@ static int guc_request_alloc(struct i915_request *rq) if (unlikely(ret < 0)) return ret; if (context_needs_register(ce, !!ret)) { - ret = guc_lrc_desc_pin(ce, true); + ret = try_context_registration(ce, true); if (unlikely(ret)) { /* unwind */ if (ret == -EPIPE) { disable_submission(guc); @@ -3560,7 +3756,7 @@ static void guc_sanitize(struct intel_engine_cs *engine) sanitize_hwsp(engine); /* And scrub the dirty cachelines for the HWSP */ - clflush_cache_range(engine->status_page.addr, PAGE_SIZE); + drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE); intel_engine_reset_pinned_contexts(engine); } @@ -3595,7 +3791,7 @@ static int guc_resume(struct intel_engine_cs *engine) setup_hwsp(engine); start_engine(engine); - if (engine->class == RENDER_CLASS) + if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) xehp_enable_ccs_engines(engine); return 0; @@ -3614,9 +3810,17 @@ static void guc_set_default_submission(struct intel_engine_cs *engine) static inline void guc_kernel_context_pin(struct intel_guc *guc, struct intel_context *ce) { + /* + * Note: we purposefully do not check the returns below because + * the registration can only fail if a reset is just starting. + * This is called at the end of reset so presumably another reset + * isn't happening and even it did this code would be run again. + */ + if (context_guc_id_invalid(ce)) pin_guc_id(guc, ce); - guc_lrc_desc_pin(ce, true); + + try_context_registration(ce, true); } static inline void guc_init_lrc_mapping(struct intel_guc *guc) @@ -3634,13 +3838,7 @@ static inline void guc_init_lrc_mapping(struct intel_guc *guc) * Also, after a reset the of the GuC we want to make sure that the * information shared with GuC is properly reset. The kernel LRCs are * not attached to the gem_context, so they need to be added separately. - * - * Note: we purposefully do not check the return of guc_lrc_desc_pin, - * because that function can only fail if a reset is just starting. This - * is at the end of reset so presumably another reset isn't happening - * and even it did this code would be run again. */ - for_each_engine(engine, gt, id) { struct intel_context *ce; @@ -3680,7 +3878,7 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine) engine->sched_engine->schedule = i915_schedule; - engine->reset.prepare = guc_reset_nop; + engine->reset.prepare = guc_engine_reset_prepare; engine->reset.rewind = guc_rewind_nop; engine->reset.cancel = guc_reset_nop; engine->reset.finish = guc_reset_nop; @@ -3699,6 +3897,10 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine) engine->flags |= I915_ENGINE_HAS_PREEMPTION; engine->flags |= I915_ENGINE_HAS_TIMESLICES; + /* Wa_14014475959:dg2 */ + if (IS_DG2(engine->i915) && engine->class == COMPUTE_CLASS) + engine->flags |= I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT; + /* * TODO: GuC supports timeslicing and semaphores as well, but they're * handled by the firmware so some minor tweaks are required before @@ -3835,32 +4037,32 @@ void intel_guc_submission_init_early(struct intel_guc *guc) spin_lock_init(&guc->timestamp.lock); INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping); - guc->submission_state.num_guc_ids = GUC_MAX_LRC_DESCRIPTORS; + guc->submission_state.num_guc_ids = GUC_MAX_CONTEXT_ID; guc->submission_supported = __guc_submission_supported(guc); guc->submission_selected = __guc_submission_selected(guc); } static inline struct intel_context * -g2h_context_lookup(struct intel_guc *guc, u32 desc_idx) +g2h_context_lookup(struct intel_guc *guc, u32 ctx_id) { struct intel_context *ce; - if (unlikely(desc_idx >= GUC_MAX_LRC_DESCRIPTORS)) { + if (unlikely(ctx_id >= GUC_MAX_CONTEXT_ID)) { drm_err(&guc_to_gt(guc)->i915->drm, - "Invalid desc_idx %u", desc_idx); + "Invalid ctx_id %u\n", ctx_id); return NULL; } - ce = __get_context(guc, desc_idx); + ce = __get_context(guc, ctx_id); if (unlikely(!ce)) { drm_err(&guc_to_gt(guc)->i915->drm, - "Context is NULL, desc_idx %u", desc_idx); + "Context is NULL, ctx_id %u\n", ctx_id); return NULL; } if (unlikely(intel_context_is_child(ce))) { drm_err(&guc_to_gt(guc)->i915->drm, - "Context is child, desc_idx %u", desc_idx); + "Context is child, ctx_id %u\n", ctx_id); return NULL; } @@ -3872,14 +4074,15 @@ int intel_guc_deregister_done_process_msg(struct intel_guc *guc, u32 len) { struct intel_context *ce; - u32 desc_idx = msg[0]; + u32 ctx_id; if (unlikely(len < 1)) { - drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len); + drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u\n", len); return -EPROTO; } + ctx_id = msg[0]; - ce = g2h_context_lookup(guc, desc_idx); + ce = g2h_context_lookup(guc, ctx_id); if (unlikely(!ce)) return -EPROTO; @@ -3923,14 +4126,15 @@ int intel_guc_sched_done_process_msg(struct intel_guc *guc, { struct intel_context *ce; unsigned long flags; - u32 desc_idx = msg[0]; + u32 ctx_id; if (unlikely(len < 2)) { - drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len); + drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u\n", len); return -EPROTO; } + ctx_id = msg[0]; - ce = g2h_context_lookup(guc, desc_idx); + ce = g2h_context_lookup(guc, ctx_id); if (unlikely(!ce)) return -EPROTO; @@ -3938,8 +4142,8 @@ int intel_guc_sched_done_process_msg(struct intel_guc *guc, (!context_pending_enable(ce) && !context_pending_disable(ce)))) { drm_err(&guc_to_gt(guc)->i915->drm, - "Bad context sched_state 0x%x, desc_idx %u", - ce->guc_state.sched_state, desc_idx); + "Bad context sched_state 0x%x, ctx_id %u\n", + ce->guc_state.sched_state, ctx_id); return -EPROTO; } @@ -4005,7 +4209,7 @@ static void capture_error_state(struct intel_guc *guc, intel_engine_set_hung_context(engine, ce); with_intel_runtime_pm(&i915->runtime_pm, wakeref) - i915_capture_error_state(gt, engine->mask); + i915_capture_error_state(gt, engine->mask, CORE_DUMP_FLAG_IS_GUC_CAPTURE); atomic_inc(&i915->gpu_error.reset_engine_count[engine->uabi_class]); } @@ -4037,14 +4241,14 @@ int intel_guc_context_reset_process_msg(struct intel_guc *guc, { struct intel_context *ce; unsigned long flags; - int desc_idx; + int ctx_id; if (unlikely(len != 1)) { drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len); return -EPROTO; } - desc_idx = msg[0]; + ctx_id = msg[0]; /* * The context lookup uses the xarray but lookups only require an RCU lock @@ -4053,7 +4257,7 @@ int intel_guc_context_reset_process_msg(struct intel_guc *guc, * asynchronously until the reset is done. */ xa_lock_irqsave(&guc->context_lookup, flags); - ce = g2h_context_lookup(guc, desc_idx); + ce = g2h_context_lookup(guc, ctx_id); if (ce) intel_context_get(ce); xa_unlock_irqrestore(&guc->context_lookup, flags); @@ -4070,23 +4274,24 @@ int intel_guc_context_reset_process_msg(struct intel_guc *guc, int intel_guc_error_capture_process_msg(struct intel_guc *guc, const u32 *msg, u32 len) { - int status; + u32 status; if (unlikely(len != 1)) { drm_dbg(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len); return -EPROTO; } - status = msg[0]; - drm_info(&guc_to_gt(guc)->i915->drm, "Got error capture: status = %d", status); + status = msg[0] & INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_MASK; + if (status == INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE) + drm_warn(&guc_to_gt(guc)->i915->drm, "G2H-Error capture no space"); - /* FIXME: Do something with the capture */ + intel_guc_capture_process(guc); return 0; } -static struct intel_engine_cs * -guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance) +struct intel_engine_cs * +intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance) { struct intel_gt *gt = guc_to_gt(guc); u8 engine_class = guc_class_to_engine_class(guc_class); @@ -4135,7 +4340,7 @@ int intel_guc_engine_failure_process_msg(struct intel_guc *guc, instance = msg[1]; reason = msg[2]; - engine = guc_lookup_engine(guc, guc_class, instance); + engine = intel_guc_lookup_engine(guc, guc_class, instance); if (unlikely(!engine)) { drm_err(>->i915->drm, "Invalid engine %d:%d", guc_class, instance); @@ -4333,17 +4538,17 @@ void intel_guc_submission_print_context_info(struct intel_guc *guc, guc_log_context_priority(p, ce); if (intel_context_is_parent(ce)) { - struct guc_process_desc *desc = __get_process_desc(ce); + struct guc_sched_wq_desc *wq_desc = __get_wq_desc(ce); struct intel_context *child; drm_printf(p, "\t\tNumber children: %u\n", ce->parallel.number_children); drm_printf(p, "\t\tWQI Head: %u\n", - READ_ONCE(desc->head)); + READ_ONCE(wq_desc->head)); drm_printf(p, "\t\tWQI Tail: %u\n", - READ_ONCE(desc->tail)); + READ_ONCE(wq_desc->tail)); drm_printf(p, "\t\tWQI Status: %u\n\n", - READ_ONCE(desc->wq_status)); + READ_ONCE(wq_desc->wq_status)); if (ce->engine->emit_bb_start == emit_bb_start_parent_no_preempt_mid_batch) { |