scsi: target: Allow userspace to request direct submissions

This allows userspace to request the fabric drivers do direct submissions
if they support it. With the new device file, submit_type, users can
write 0 - 2 to control how commands are submitted to the backend:

 0 - TARGET_FABRIC_DEFAULT_SUBMIT - LIO will use the fabric's default
     submission type. This is the default for compat.

 1 - TARGET_DIRECT_SUBMIT - LIO will submit the cmd to the backend from the
     calling context if the fabric the cmd was received on supports it,
     else it will use the fabric's default type.

 2 - TARGET_QUEUE_SUBMIT - LIO will queue the cmd to the LIO submission
     workqueue which will pass it to the backend.

When using an NVMe drive and vhost-scsi with direct submission we see
around a 20% improvement in 4K I/Os:

fio jobs        1       2       4       8       10
--------------------------------------------------
defer           94K     190K    394K    770K    890K
direct          128K    252K    488K    950K    -

And when using the queueing mode, we now no longer see issues like where
the iSCSI tx thread is blocked in the block layer waiting on a tag so it
can't respond to a nop or perform I/Os for other LUs.

Signed-off-by: Mike Christie <michael.christie@oracle.com>
Link: https://lore.kernel.org/r/20230928020907.5730-6-michael.christie@oracle.com
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

1 files changed, 31 insertions, 10 deletions

diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 6c8f6055fc1e..f1fab7e0ea3d 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -1575,14 +1575,7 @@ target_cmd_parse_cdb(struct se_cmd *cmd)
 }
 EXPORT_SYMBOL(target_cmd_parse_cdb);
 
-/**
- * target_submit - perform final initialization and submit cmd to LIO core
- * @cmd: command descriptor to submit
- *
- * target_submit_prep or something similar must have been called on the cmd,
- * and this must be called from process context.
- */
-int target_submit(struct se_cmd *cmd)
+static int __target_submit(struct se_cmd *cmd)
 {
 	sense_reason_t ret;
 
@@ -1642,7 +1635,6 @@ int target_submit(struct se_cmd *cmd)
 		transport_generic_request_failure(cmd, ret);
 	return 0;
 }
-EXPORT_SYMBOL_GPL(target_submit);
 
 sense_reason_t
 transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
@@ -1904,7 +1896,7 @@ void target_queued_submit_work(struct work_struct *work)
 			se_plug = target_plug_device(se_dev);
 		}
 
-		target_submit(se_cmd);
+		__target_submit(se_cmd);
 	}
 
 	if (se_plug)
@@ -1927,6 +1919,35 @@ void target_queue_submission(struct se_cmd *se_cmd)
 }
 EXPORT_SYMBOL_GPL(target_queue_submission);
 
+/**
+ * target_submit - perform final initialization and submit cmd to LIO core
+ * @cmd: command descriptor to submit
+ *
+ * target_submit_prep or something similar must have been called on the cmd,
+ * and this must be called from process context.
+ */
+int target_submit(struct se_cmd *se_cmd)
+{
+	const struct target_core_fabric_ops *tfo = se_cmd->se_sess->se_tpg->se_tpg_tfo;
+	struct se_dev_attrib *da = &se_cmd->se_dev->dev_attrib;
+	u8 submit_type;
+
+	if (da->submit_type == TARGET_FABRIC_DEFAULT_SUBMIT)
+		submit_type = tfo->default_submit_type;
+	else if (da->submit_type == TARGET_DIRECT_SUBMIT &&
+		 tfo->direct_submit_supp)
+		submit_type = TARGET_DIRECT_SUBMIT;
+	else
+		submit_type = TARGET_QUEUE_SUBMIT;
+
+	if (submit_type == TARGET_DIRECT_SUBMIT)
+		return __target_submit(se_cmd);
+
+	target_queue_submission(se_cmd);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(target_submit);
+
 static void target_complete_tmr_failure(struct work_struct *work)
 {
 	struct se_cmd *se_cmd = container_of(work, struct se_cmd, work);