diff options
Diffstat (limited to 'drivers/misc/habanalabs/common/command_submission.c')
-rw-r--r-- | drivers/misc/habanalabs/common/command_submission.c | 473 |
1 files changed, 423 insertions, 50 deletions
diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index b2b3d2b0f808..7bd4a03b3429 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -48,8 +48,8 @@ void hl_sob_reset_error(struct kref *ref) struct hl_device *hdev = hw_sob->hdev; dev_crit(hdev->dev, - "SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n", - hw_sob->q_idx, hw_sob->sob_id); + "SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n", + hw_sob->q_idx, hw_sob->sob_id); } /** @@ -149,9 +149,10 @@ void hl_fence_get(struct hl_fence *fence) kref_get(&fence->refcount); } -static void hl_fence_init(struct hl_fence *fence) +static void hl_fence_init(struct hl_fence *fence, u64 sequence) { kref_init(&fence->refcount); + fence->cs_sequence = sequence; fence->error = 0; fence->timestamp = ktime_set(0, 0); init_completion(&fence->completion); @@ -184,6 +185,28 @@ static void cs_job_put(struct hl_cs_job *job) kref_put(&job->refcount, cs_job_do_release); } +bool cs_needs_completion(struct hl_cs *cs) +{ + /* In case this is a staged CS, only the last CS in sequence should + * get a completion, any non staged CS will always get a completion + */ + if (cs->staged_cs && !cs->staged_last) + return false; + + return true; +} + +bool cs_needs_timeout(struct hl_cs *cs) +{ + /* In case this is a staged CS, only the first CS in sequence should + * get a timeout, any non staged CS will always get a timeout + */ + if (cs->staged_cs && !cs->staged_first) + return false; + + return true; +} + static bool is_cb_patched(struct hl_device *hdev, struct hl_cs_job *job) { /* @@ -225,6 +248,7 @@ static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job) parser.queue_type = job->queue_type; parser.is_kernel_allocated_cb = job->is_kernel_allocated_cb; job->patched_cb = NULL; + parser.completion = cs_needs_completion(job->cs); rc = hdev->asic_funcs->cs_parser(hdev, &parser); @@ -290,13 +314,153 @@ static void complete_job(struct hl_device *hdev, struct hl_cs_job *job) hl_debugfs_remove_job(hdev, job); - if (job->queue_type == QUEUE_TYPE_EXT || - job->queue_type == QUEUE_TYPE_HW) + /* We decrement reference only for a CS that gets completion + * because the reference was incremented only for this kind of CS + * right before it was scheduled. + * + * In staged submission, only the last CS marked as 'staged_last' + * gets completion, hence its release function will be called from here. + * As for all the rest CS's in the staged submission which do not get + * completion, their CS reference will be decremented by the + * 'staged_last' CS during the CS release flow. + * All relevant PQ CI counters will be incremented during the CS release + * flow by calling 'hl_hw_queue_update_ci'. + */ + if (cs_needs_completion(cs) && + (job->queue_type == QUEUE_TYPE_EXT || + job->queue_type == QUEUE_TYPE_HW)) cs_put(cs); cs_job_put(job); } +/* + * hl_staged_cs_find_first - locate the first CS in this staged submission + * + * @hdev: pointer to device structure + * @cs_seq: staged submission sequence number + * + * @note: This function must be called under 'hdev->cs_mirror_lock' + * + * Find and return a CS pointer with the given sequence + */ +struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq) +{ + struct hl_cs *cs; + + list_for_each_entry_reverse(cs, &hdev->cs_mirror_list, mirror_node) + if (cs->staged_cs && cs->staged_first && + cs->sequence == cs_seq) + return cs; + + return NULL; +} + +/* + * is_staged_cs_last_exists - returns true if the last CS in sequence exists + * + * @hdev: pointer to device structure + * @cs: staged submission member + * + */ +bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs) +{ + struct hl_cs *last_entry; + + last_entry = list_last_entry(&cs->staged_cs_node, struct hl_cs, + staged_cs_node); + + if (last_entry->staged_last) + return true; + + return false; +} + +/* + * staged_cs_get - get CS reference if this CS is a part of a staged CS + * + * @hdev: pointer to device structure + * @cs: current CS + * @cs_seq: staged submission sequence number + * + * Increment CS reference for every CS in this staged submission except for + * the CS which get completion. + */ +static void staged_cs_get(struct hl_device *hdev, struct hl_cs *cs) +{ + /* Only the last CS in this staged submission will get a completion. + * We must increment the reference for all other CS's in this + * staged submission. + * Once we get a completion we will release the whole staged submission. + */ + if (!cs->staged_last) + cs_get(cs); +} + +/* + * staged_cs_put - put a CS in case it is part of staged submission + * + * @hdev: pointer to device structure + * @cs: CS to put + * + * This function decrements a CS reference (for a non completion CS) + */ +static void staged_cs_put(struct hl_device *hdev, struct hl_cs *cs) +{ + /* We release all CS's in a staged submission except the last + * CS which we have never incremented its reference. + */ + if (!cs_needs_completion(cs)) + cs_put(cs); +} + +static void cs_handle_tdr(struct hl_device *hdev, struct hl_cs *cs) +{ + bool next_entry_found = false; + struct hl_cs *next; + + if (!cs_needs_timeout(cs)) + return; + + spin_lock(&hdev->cs_mirror_lock); + + /* We need to handle tdr only once for the complete staged submission. + * Hence, we choose the CS that reaches this function first which is + * the CS marked as 'staged_last'. + */ + if (cs->staged_cs && cs->staged_last) + cs = hl_staged_cs_find_first(hdev, cs->staged_sequence); + + spin_unlock(&hdev->cs_mirror_lock); + + /* Don't cancel TDR in case this CS was timedout because we might be + * running from the TDR context + */ + if (cs && (cs->timedout || + hdev->timeout_jiffies == MAX_SCHEDULE_TIMEOUT)) + return; + + if (cs && cs->tdr_active) + cancel_delayed_work_sync(&cs->work_tdr); + + spin_lock(&hdev->cs_mirror_lock); + + /* queue TDR for next CS */ + list_for_each_entry(next, &hdev->cs_mirror_list, mirror_node) + if (cs_needs_timeout(next)) { + next_entry_found = true; + break; + } + + if (next_entry_found && !next->tdr_active) { + next->tdr_active = true; + schedule_delayed_work(&next->work_tdr, + hdev->timeout_jiffies); + } + + spin_unlock(&hdev->cs_mirror_lock); +} + static void cs_do_release(struct kref *ref) { struct hl_cs *cs = container_of(ref, struct hl_cs, refcount); @@ -346,36 +510,37 @@ static void cs_do_release(struct kref *ref) hdev->asic_funcs->hw_queues_unlock(hdev); - /* Need to update CI for internal queues */ - hl_int_hw_queue_update_ci(cs); + /* Need to update CI for all queue jobs that does not get completion */ + hl_hw_queue_update_ci(cs); /* remove CS from CS mirror list */ spin_lock(&hdev->cs_mirror_lock); list_del_init(&cs->mirror_node); spin_unlock(&hdev->cs_mirror_lock); - /* Don't cancel TDR in case this CS was timedout because we might be - * running from the TDR context - */ - if (!cs->timedout && hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) { - struct hl_cs *next; - - if (cs->tdr_active) - cancel_delayed_work_sync(&cs->work_tdr); + cs_handle_tdr(hdev, cs); - spin_lock(&hdev->cs_mirror_lock); - - /* queue TDR for next CS */ - next = list_first_entry_or_null(&hdev->cs_mirror_list, - struct hl_cs, mirror_node); + if (cs->staged_cs) { + /* the completion CS decrements reference for the entire + * staged submission + */ + if (cs->staged_last) { + struct hl_cs *staged_cs, *tmp; - if (next && !next->tdr_active) { - next->tdr_active = true; - schedule_delayed_work(&next->work_tdr, - hdev->timeout_jiffies); + list_for_each_entry_safe(staged_cs, tmp, + &cs->staged_cs_node, staged_cs_node) + staged_cs_put(hdev, staged_cs); } - spin_unlock(&hdev->cs_mirror_lock); + /* A staged CS will be a member in the list only after it + * was submitted. We used 'cs_mirror_lock' when inserting + * it to list so we will use it again when removing it + */ + if (cs->submitted) { + spin_lock(&hdev->cs_mirror_lock); + list_del(&cs->staged_cs_node); + spin_unlock(&hdev->cs_mirror_lock); + } } out: @@ -461,7 +626,8 @@ static void cs_timedout(struct work_struct *work) } static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, - enum hl_cs_type cs_type, struct hl_cs **cs_new) + enum hl_cs_type cs_type, u64 user_sequence, + struct hl_cs **cs_new) { struct hl_cs_counters_atomic *cntr; struct hl_fence *other = NULL; @@ -478,6 +644,9 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, return -ENOMEM; } + /* increment refcnt for context */ + hl_ctx_get(hdev, ctx); + cs->ctx = ctx; cs->submitted = false; cs->completed = false; @@ -507,6 +676,18 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, (hdev->asic_prop.max_pending_cs - 1)]; if (other && !completion_done(&other->completion)) { + /* If the following statement is true, it means we have reached + * a point in which only part of the staged submission was + * submitted and we don't have enough room in the 'cs_pending' + * array for the rest of the submission. + * This causes a deadlock because this CS will never be + * completed as it depends on future CS's for completion. + */ + if (other->cs_sequence == user_sequence) + dev_crit_ratelimited(hdev->dev, + "Staged CS %llu deadlock due to lack of resources", + user_sequence); + dev_dbg_ratelimited(hdev->dev, "Rejecting CS because of too many in-flights CS\n"); atomic64_inc(&ctx->cs_counters.max_cs_in_flight_drop_cnt); @@ -525,7 +706,7 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, } /* init hl_fence */ - hl_fence_init(&cs_cmpl->base_fence); + hl_fence_init(&cs_cmpl->base_fence, cs_cmpl->cs_seq); cs->sequence = cs_cmpl->cs_seq; @@ -549,6 +730,7 @@ free_fence: kfree(cs_cmpl); free_cs: kfree(cs); + hl_ctx_put(ctx); return rc; } @@ -556,6 +738,8 @@ static void cs_rollback(struct hl_device *hdev, struct hl_cs *cs) { struct hl_cs_job *job, *tmp; + staged_cs_put(hdev, cs); + list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node) complete_job(hdev, job); } @@ -565,7 +749,9 @@ void hl_cs_rollback_all(struct hl_device *hdev) int i; struct hl_cs *cs, *tmp; - /* flush all completions */ + /* flush all completions before iterating over the CS mirror list in + * order to avoid a race with the release functions + */ for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) flush_workqueue(hdev->cq_wq[i]); @@ -574,12 +760,24 @@ void hl_cs_rollback_all(struct hl_device *hdev) cs_get(cs); cs->aborted = true; dev_warn_ratelimited(hdev->dev, "Killing CS %d.%llu\n", - cs->ctx->asid, cs->sequence); + cs->ctx->asid, cs->sequence); cs_rollback(hdev, cs); cs_put(cs); } } +void hl_pending_cb_list_flush(struct hl_ctx *ctx) +{ + struct hl_pending_cb *pending_cb, *tmp; + + list_for_each_entry_safe(pending_cb, tmp, + &ctx->pending_cb_list, cb_node) { + list_del(&pending_cb->cb_node); + hl_cb_put(pending_cb->cb); + kfree(pending_cb); + } +} + static void job_wq_completion(struct work_struct *work) { struct hl_cs_job *job = container_of(work, struct hl_cs_job, @@ -734,6 +932,12 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args) return -EBUSY; } + if ((args->in.cs_flags & HL_CS_FLAGS_STAGED_SUBMISSION) && + !hdev->supports_staged_submission) { + dev_err(hdev->dev, "staged submission not supported"); + return -EPERM; + } + cs_type_flags = args->in.cs_flags & HL_CS_FLAGS_TYPE_MASK; if (unlikely(cs_type_flags && !is_power_of_2(cs_type_flags))) { @@ -805,10 +1009,38 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev, return 0; } +static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs, + u64 sequence, u32 flags) +{ + if (!(flags & HL_CS_FLAGS_STAGED_SUBMISSION)) + return 0; + + cs->staged_last = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_LAST); + cs->staged_first = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST); + + if (cs->staged_first) { + /* Staged CS sequence is the first CS sequence */ + INIT_LIST_HEAD(&cs->staged_cs_node); + cs->staged_sequence = cs->sequence; + } else { + /* User sequence will be validated in 'hl_hw_queue_schedule_cs' + * under the cs_mirror_lock + */ + cs->staged_sequence = sequence; + } + + /* Increment CS reference if needed */ + staged_cs_get(hdev, cs); + + cs->staged_cs = true; + + return 0; +} + static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, - u32 num_chunks, u64 *cs_seq, bool timestamp) + u32 num_chunks, u64 *cs_seq, u32 flags) { - bool int_queues_only = true; + bool staged_mid, int_queues_only = true; struct hl_device *hdev = hpriv->hdev; struct hl_cs_chunk *cs_chunk_array; struct hl_cs_counters_atomic *cntr; @@ -816,9 +1048,11 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, struct hl_cs_job *job; struct hl_cs *cs; struct hl_cb *cb; + u64 user_sequence; int rc, i; cntr = &hdev->aggregated_cs_counters; + user_sequence = *cs_seq; *cs_seq = ULLONG_MAX; rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks, @@ -826,20 +1060,26 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, if (rc) goto out; - /* increment refcnt for context */ - hl_ctx_get(hdev, hpriv->ctx); + if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) && + !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST)) + staged_mid = true; + else + staged_mid = false; - rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, &cs); - if (rc) { - hl_ctx_put(hpriv->ctx); + rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, + staged_mid ? user_sequence : ULLONG_MAX, &cs); + if (rc) goto free_cs_chunk_array; - } - cs->timestamp = !!timestamp; + cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP); *cs_seq = cs->sequence; hl_debugfs_add_cs(cs); + rc = cs_staged_submission(hdev, cs, user_sequence, flags); + if (rc) + goto free_cs_object; + /* Validate ALL the CS chunks before submitting the CS */ for (i = 0 ; i < num_chunks ; i++) { struct hl_cs_chunk *chunk = &cs_chunk_array[i]; @@ -899,8 +1139,9 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, * Only increment for JOB on external or H/W queues, because * only for those JOBs we get completion */ - if (job->queue_type == QUEUE_TYPE_EXT || - job->queue_type == QUEUE_TYPE_HW) + if (cs_needs_completion(cs) && + (job->queue_type == QUEUE_TYPE_EXT || + job->queue_type == QUEUE_TYPE_HW)) cs_get(cs); hl_debugfs_add_job(hdev, job); @@ -916,11 +1157,14 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, } } - if (int_queues_only) { + /* We allow a CS with any queue type combination as long as it does + * not get a completion + */ + if (int_queues_only && cs_needs_completion(cs)) { atomic64_inc(&ctx->cs_counters.validation_drop_cnt); atomic64_inc(&cntr->validation_drop_cnt); dev_err(hdev->dev, - "Reject CS %d.%llu because only internal queues jobs are present\n", + "Reject CS %d.%llu since it contains only internal queues jobs and needs completion\n", cs->ctx->asid, cs->sequence); rc = -EINVAL; goto free_cs_object; @@ -954,6 +1198,129 @@ out: return rc; } +static int pending_cb_create_job(struct hl_device *hdev, struct hl_ctx *ctx, + struct hl_cs *cs, struct hl_cb *cb, u32 size, u32 hw_queue_id) +{ + struct hw_queue_properties *hw_queue_prop; + struct hl_cs_counters_atomic *cntr; + struct hl_cs_job *job; + + hw_queue_prop = &hdev->asic_prop.hw_queues_props[hw_queue_id]; + cntr = &hdev->aggregated_cs_counters; + + job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true); + if (!job) { + atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); + atomic64_inc(&cntr->out_of_mem_drop_cnt); + dev_err(hdev->dev, "Failed to allocate a new job\n"); + return -ENOMEM; + } + + job->id = 0; + job->cs = cs; + job->user_cb = cb; + atomic_inc(&job->user_cb->cs_cnt); + job->user_cb_size = size; + job->hw_queue_id = hw_queue_id; + job->patched_cb = job->user_cb; + job->job_cb_size = job->user_cb_size; + + /* increment refcount as for external queues we get completion */ + cs_get(cs); + + cs->jobs_in_queue_cnt[job->hw_queue_id]++; + + list_add_tail(&job->cs_node, &cs->job_list); + + hl_debugfs_add_job(hdev, job); + + return 0; +} + +static int hl_submit_pending_cb(struct hl_fpriv *hpriv) +{ + struct hl_device *hdev = hpriv->hdev; + struct hl_ctx *ctx = hpriv->ctx; + struct hl_pending_cb *pending_cb, *tmp; + struct list_head local_cb_list; + struct hl_cs *cs; + struct hl_cb *cb; + u32 hw_queue_id; + u32 cb_size; + int process_list, rc = 0; + + if (list_empty(&ctx->pending_cb_list)) + return 0; + + process_list = atomic_cmpxchg(&ctx->thread_pending_cb_token, 1, 0); + + /* Only a single thread is allowed to process the list */ + if (!process_list) + return 0; + + if (list_empty(&ctx->pending_cb_list)) + goto free_pending_cb_token; + + /* move all list elements to a local list */ + INIT_LIST_HEAD(&local_cb_list); + spin_lock(&ctx->pending_cb_lock); + list_for_each_entry_safe(pending_cb, tmp, &ctx->pending_cb_list, + cb_node) + list_move_tail(&pending_cb->cb_node, &local_cb_list); + spin_unlock(&ctx->pending_cb_lock); + + rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, ULLONG_MAX, &cs); + if (rc) + goto add_list_elements; + + hl_debugfs_add_cs(cs); + + /* Iterate through pending cb list, create jobs and add to CS */ + list_for_each_entry(pending_cb, &local_cb_list, cb_node) { + cb = pending_cb->cb; + cb_size = pending_cb->cb_size; + hw_queue_id = pending_cb->hw_queue_id; + + rc = pending_cb_create_job(hdev, ctx, cs, cb, cb_size, + hw_queue_id); + if (rc) + goto free_cs_object; + } + + rc = hl_hw_queue_schedule_cs(cs); + if (rc) { + if (rc != -EAGAIN) + dev_err(hdev->dev, + "Failed to submit CS %d.%llu (%d)\n", + ctx->asid, cs->sequence, rc); + goto free_cs_object; + } + + /* pending cb was scheduled successfully */ + list_for_each_entry_safe(pending_cb, tmp, &local_cb_list, cb_node) { + list_del(&pending_cb->cb_node); + kfree(pending_cb); + } + + cs_put(cs); + + goto free_pending_cb_token; + +free_cs_object: + cs_rollback(hdev, cs); + cs_put(cs); +add_list_elements: + spin_lock(&ctx->pending_cb_lock); + list_for_each_entry_safe_reverse(pending_cb, tmp, &local_cb_list, + cb_node) + list_move(&pending_cb->cb_node, &ctx->pending_cb_list); + spin_unlock(&ctx->pending_cb_lock); +free_pending_cb_token: + atomic_set(&ctx->thread_pending_cb_token, 1); + + return rc; +} + static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, u64 *cs_seq) { @@ -1003,7 +1370,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, rc = 0; } else { rc = cs_ioctl_default(hpriv, chunks, num_chunks, - cs_seq, false); + cs_seq, 0); } mutex_unlock(&hpriv->restore_phase_mutex); @@ -1275,15 +1642,11 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, } } - /* increment refcnt for context */ - hl_ctx_get(hdev, ctx); - - rc = allocate_cs(hdev, ctx, cs_type, &cs); + rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs); if (rc) { if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT) hl_fence_put(sig_fence); - hl_ctx_put(ctx); goto free_cs_chunk_array; } @@ -1346,7 +1709,7 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) enum hl_cs_type cs_type; u64 cs_seq = ULONG_MAX; void __user *chunks; - u32 num_chunks; + u32 num_chunks, flags; int rc; rc = hl_cs_sanity_checks(hpriv, args); @@ -1357,10 +1720,20 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) if (rc) goto out; + rc = hl_submit_pending_cb(hpriv); + if (rc) + goto out; + cs_type = hl_cs_get_cs_type(args->in.cs_flags & ~HL_CS_FLAGS_FORCE_RESTORE); chunks = (void __user *) (uintptr_t) args->in.chunks_execute; num_chunks = args->in.num_chunks_execute; + flags = args->in.cs_flags; + + /* In case this is a staged CS, user should supply the CS sequence */ + if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) && + !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST)) + cs_seq = args->in.seq; switch (cs_type) { case CS_TYPE_SIGNAL: @@ -1371,7 +1744,7 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) break; default: rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq, - args->in.cs_flags & HL_CS_FLAGS_TIMESTAMP); + args->in.cs_flags); break; } |