diff options
Diffstat (limited to 'drivers/target/target_core_transport.c')
-rw-r--r-- | drivers/target/target_core_transport.c | 199 |
1 files changed, 128 insertions, 71 deletions
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 5926316252eb..86adff2a86ed 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -220,12 +220,52 @@ void transport_subsystem_check_init(void) sub_api_initialized = 1; } -static void target_release_sess_cmd_refcnt(struct percpu_ref *ref) +static void target_release_cmd_refcnt(struct percpu_ref *ref) { - struct se_session *sess = container_of(ref, typeof(*sess), cmd_count); + struct target_cmd_counter *cmd_cnt = container_of(ref, + typeof(*cmd_cnt), + refcnt); + wake_up(&cmd_cnt->refcnt_wq); +} + +struct target_cmd_counter *target_alloc_cmd_counter(void) +{ + struct target_cmd_counter *cmd_cnt; + int rc; + + cmd_cnt = kzalloc(sizeof(*cmd_cnt), GFP_KERNEL); + if (!cmd_cnt) + return NULL; + + init_completion(&cmd_cnt->stop_done); + init_waitqueue_head(&cmd_cnt->refcnt_wq); + atomic_set(&cmd_cnt->stopped, 0); - wake_up(&sess->cmd_count_wq); + rc = percpu_ref_init(&cmd_cnt->refcnt, target_release_cmd_refcnt, 0, + GFP_KERNEL); + if (rc) + goto free_cmd_cnt; + + return cmd_cnt; + +free_cmd_cnt: + kfree(cmd_cnt); + return NULL; } +EXPORT_SYMBOL_GPL(target_alloc_cmd_counter); + +void target_free_cmd_counter(struct target_cmd_counter *cmd_cnt) +{ + /* + * Drivers like loop do not call target_stop_session during session + * shutdown so we have to drop the ref taken at init time here. + */ + if (!atomic_read(&cmd_cnt->stopped)) + percpu_ref_put(&cmd_cnt->refcnt); + + percpu_ref_exit(&cmd_cnt->refcnt); +} +EXPORT_SYMBOL_GPL(target_free_cmd_counter); /** * transport_init_session - initialize a session object @@ -233,32 +273,14 @@ static void target_release_sess_cmd_refcnt(struct percpu_ref *ref) * * The caller must have zero-initialized @se_sess before calling this function. */ -int transport_init_session(struct se_session *se_sess) +void transport_init_session(struct se_session *se_sess) { INIT_LIST_HEAD(&se_sess->sess_list); INIT_LIST_HEAD(&se_sess->sess_acl_list); spin_lock_init(&se_sess->sess_cmd_lock); - init_waitqueue_head(&se_sess->cmd_count_wq); - init_completion(&se_sess->stop_done); - atomic_set(&se_sess->stopped, 0); - return percpu_ref_init(&se_sess->cmd_count, - target_release_sess_cmd_refcnt, 0, GFP_KERNEL); } EXPORT_SYMBOL(transport_init_session); -void transport_uninit_session(struct se_session *se_sess) -{ - /* - * Drivers like iscsi and loop do not call target_stop_session - * during session shutdown so we have to drop the ref taken at init - * time here. - */ - if (!atomic_read(&se_sess->stopped)) - percpu_ref_put(&se_sess->cmd_count); - - percpu_ref_exit(&se_sess->cmd_count); -} - /** * transport_alloc_session - allocate a session object and initialize it * @sup_prot_ops: bitmask that defines which T10-PI modes are supported. @@ -266,7 +288,6 @@ void transport_uninit_session(struct se_session *se_sess) struct se_session *transport_alloc_session(enum target_prot_op sup_prot_ops) { struct se_session *se_sess; - int ret; se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL); if (!se_sess) { @@ -274,11 +295,7 @@ struct se_session *transport_alloc_session(enum target_prot_op sup_prot_ops) " se_sess_cache\n"); return ERR_PTR(-ENOMEM); } - ret = transport_init_session(se_sess); - if (ret < 0) { - kmem_cache_free(se_sess_cache, se_sess); - return ERR_PTR(ret); - } + transport_init_session(se_sess); se_sess->sup_prot_ops = sup_prot_ops; return se_sess; @@ -444,8 +461,13 @@ target_setup_session(struct se_portal_group *tpg, int (*callback)(struct se_portal_group *, struct se_session *, void *)) { + struct target_cmd_counter *cmd_cnt; struct se_session *sess; + int rc; + cmd_cnt = target_alloc_cmd_counter(); + if (!cmd_cnt) + return ERR_PTR(-ENOMEM); /* * If the fabric driver is using percpu-ida based pre allocation * of I/O descriptor tags, go ahead and perform that setup now.. @@ -455,29 +477,36 @@ target_setup_session(struct se_portal_group *tpg, else sess = transport_alloc_session(prot_op); - if (IS_ERR(sess)) - return sess; + if (IS_ERR(sess)) { + rc = PTR_ERR(sess); + goto free_cnt; + } + sess->cmd_cnt = cmd_cnt; sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg, (unsigned char *)initiatorname); if (!sess->se_node_acl) { - transport_free_session(sess); - return ERR_PTR(-EACCES); + rc = -EACCES; + goto free_sess; } /* * Go ahead and perform any remaining fabric setup that is * required before transport_register_session(). */ if (callback != NULL) { - int rc = callback(tpg, sess, private); - if (rc) { - transport_free_session(sess); - return ERR_PTR(rc); - } + rc = callback(tpg, sess, private); + if (rc) + goto free_sess; } transport_register_session(tpg, sess->se_node_acl, sess, private); return sess; + +free_sess: + transport_free_session(sess); +free_cnt: + target_free_cmd_counter(cmd_cnt); + return ERR_PTR(rc); } EXPORT_SYMBOL(target_setup_session); @@ -602,7 +631,8 @@ void transport_free_session(struct se_session *se_sess) sbitmap_queue_free(&se_sess->sess_tag_pool); kvfree(se_sess->sess_cmd_map); } - transport_uninit_session(se_sess); + if (se_sess->cmd_cnt) + target_free_cmd_counter(se_sess->cmd_cnt); kmem_cache_free(se_sess_cache, se_sess); } EXPORT_SYMBOL(transport_free_session); @@ -1412,14 +1442,12 @@ target_cmd_size_check(struct se_cmd *cmd, unsigned int size) * * Preserves the value of @cmd->tag. */ -void __target_init_cmd( - struct se_cmd *cmd, - const struct target_core_fabric_ops *tfo, - struct se_session *se_sess, - u32 data_length, - int data_direction, - int task_attr, - unsigned char *sense_buffer, u64 unpacked_lun) +void __target_init_cmd(struct se_cmd *cmd, + const struct target_core_fabric_ops *tfo, + struct se_session *se_sess, u32 data_length, + int data_direction, int task_attr, + unsigned char *sense_buffer, u64 unpacked_lun, + struct target_cmd_counter *cmd_cnt) { INIT_LIST_HEAD(&cmd->se_delayed_node); INIT_LIST_HEAD(&cmd->se_qf_node); @@ -1439,6 +1467,7 @@ void __target_init_cmd( cmd->sam_task_attr = task_attr; cmd->sense_buffer = sense_buffer; cmd->orig_fe_lun = unpacked_lun; + cmd->cmd_cnt = cmd_cnt; if (!(cmd->se_cmd_flags & SCF_USE_CPUID)) cmd->cpuid = raw_smp_processor_id(); @@ -1658,7 +1687,8 @@ int target_init_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, * target_core_fabric_ops->queue_status() callback */ __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, data_length, - data_dir, task_attr, sense, unpacked_lun); + data_dir, task_attr, sense, unpacked_lun, + se_sess->cmd_cnt); /* * Obtain struct se_cmd->cmd_kref reference. A second kref_get here is @@ -1953,7 +1983,8 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, BUG_ON(!se_tpg); __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - 0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun); + 0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun, + se_sess->cmd_cnt); /* * FIXME: Currently expect caller to handle se_cmd->se_tmr_req * allocation failure. @@ -2957,7 +2988,6 @@ EXPORT_SYMBOL(transport_generic_free_cmd); */ int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) { - struct se_session *se_sess = se_cmd->se_sess; int ret = 0; /* @@ -2970,9 +3000,14 @@ int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) se_cmd->se_cmd_flags |= SCF_ACK_KREF; } - if (!percpu_ref_tryget_live(&se_sess->cmd_count)) - ret = -ESHUTDOWN; - + /* + * Users like xcopy do not use counters since they never do a stop + * and wait. + */ + if (se_cmd->cmd_cnt) { + if (!percpu_ref_tryget_live(&se_cmd->cmd_cnt->refcnt)) + ret = -ESHUTDOWN; + } if (ret && ack_kref) target_put_sess_cmd(se_cmd); @@ -2993,7 +3028,7 @@ static void target_free_cmd_mem(struct se_cmd *cmd) static void target_release_cmd_kref(struct kref *kref) { struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref); - struct se_session *se_sess = se_cmd->se_sess; + struct target_cmd_counter *cmd_cnt = se_cmd->cmd_cnt; struct completion *free_compl = se_cmd->free_compl; struct completion *abrt_compl = se_cmd->abrt_compl; @@ -3004,7 +3039,8 @@ static void target_release_cmd_kref(struct kref *kref) if (abrt_compl) complete(abrt_compl); - percpu_ref_put(&se_sess->cmd_count); + if (cmd_cnt) + percpu_ref_put(&cmd_cnt->refcnt); } /** @@ -3123,46 +3159,67 @@ void target_show_cmd(const char *pfx, struct se_cmd *cmd) } EXPORT_SYMBOL(target_show_cmd); -static void target_stop_session_confirm(struct percpu_ref *ref) +static void target_stop_cmd_counter_confirm(struct percpu_ref *ref) +{ + struct target_cmd_counter *cmd_cnt = container_of(ref, + struct target_cmd_counter, + refcnt); + complete_all(&cmd_cnt->stop_done); +} + +/** + * target_stop_cmd_counter - Stop new IO from being added to the counter. + * @cmd_cnt: counter to stop + */ +void target_stop_cmd_counter(struct target_cmd_counter *cmd_cnt) { - struct se_session *se_sess = container_of(ref, struct se_session, - cmd_count); - complete_all(&se_sess->stop_done); + pr_debug("Stopping command counter.\n"); + if (!atomic_cmpxchg(&cmd_cnt->stopped, 0, 1)) + percpu_ref_kill_and_confirm(&cmd_cnt->refcnt, + target_stop_cmd_counter_confirm); } +EXPORT_SYMBOL_GPL(target_stop_cmd_counter); /** * target_stop_session - Stop new IO from being queued on the session. - * @se_sess: session to stop + * @se_sess: session to stop */ void target_stop_session(struct se_session *se_sess) { - pr_debug("Stopping session queue.\n"); - if (atomic_cmpxchg(&se_sess->stopped, 0, 1) == 0) - percpu_ref_kill_and_confirm(&se_sess->cmd_count, - target_stop_session_confirm); + target_stop_cmd_counter(se_sess->cmd_cnt); } EXPORT_SYMBOL(target_stop_session); /** - * target_wait_for_sess_cmds - Wait for outstanding commands - * @se_sess: session to wait for active I/O + * target_wait_for_cmds - Wait for outstanding cmds. + * @cmd_cnt: counter to wait for active I/O for. */ -void target_wait_for_sess_cmds(struct se_session *se_sess) +void target_wait_for_cmds(struct target_cmd_counter *cmd_cnt) { int ret; - WARN_ON_ONCE(!atomic_read(&se_sess->stopped)); + WARN_ON_ONCE(!atomic_read(&cmd_cnt->stopped)); do { pr_debug("Waiting for running cmds to complete.\n"); - ret = wait_event_timeout(se_sess->cmd_count_wq, - percpu_ref_is_zero(&se_sess->cmd_count), - 180 * HZ); + ret = wait_event_timeout(cmd_cnt->refcnt_wq, + percpu_ref_is_zero(&cmd_cnt->refcnt), + 180 * HZ); } while (ret <= 0); - wait_for_completion(&se_sess->stop_done); + wait_for_completion(&cmd_cnt->stop_done); pr_debug("Waiting for cmds done.\n"); } +EXPORT_SYMBOL_GPL(target_wait_for_cmds); + +/** + * target_wait_for_sess_cmds - Wait for outstanding commands + * @se_sess: session to wait for active I/O + */ +void target_wait_for_sess_cmds(struct se_session *se_sess) +{ + target_wait_for_cmds(se_sess->cmd_cnt); +} EXPORT_SYMBOL(target_wait_for_sess_cmds); /* |