Update Linux to v5.10.109
Sourced from [1]
[1] https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.109.tar.xz
Change-Id: I19bca9fc6762d4e63bcf3e4cba88bbe560d9c76c
Signed-off-by: Olivier Deprez <olivier.deprez@arm.com>
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 8e6d7ba..6f3d29d 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -52,7 +52,6 @@
#define SCSI_INLINE_SG_CNT 2
#endif
-static struct kmem_cache *scsi_sdb_cache;
static struct kmem_cache *scsi_sense_cache;
static struct kmem_cache *scsi_sense_isadma_cache;
static DEFINE_MUTEX(scsi_sense_cache_mutex);
@@ -189,7 +188,7 @@
* active on the host/device.
*/
if (unbusy)
- scsi_device_unbusy(device);
+ scsi_device_unbusy(device, cmd);
/*
* Requeue this command. It will go before all other commands
@@ -202,24 +201,17 @@
blk_mq_requeue_request(cmd->request, true);
}
-/*
- * Function: scsi_queue_insert()
+/**
+ * scsi_queue_insert - Reinsert a command in the queue.
+ * @cmd: command that we are adding to queue.
+ * @reason: why we are inserting command to queue.
*
- * Purpose: Insert a command in the midlevel queue.
+ * We do this for one of two cases. Either the host is busy and it cannot accept
+ * any more commands for the time being, or the device returned QUEUE_FULL and
+ * can accept no more commands.
*
- * Arguments: cmd - command that we are adding to queue.
- * reason - why we are inserting command to queue.
- *
- * Lock status: Assumed that lock is not held upon entry.
- *
- * Returns: Nothing.
- *
- * Notes: We do this for one of two cases. Either the host is busy
- * and it cannot accept any more commands for the time being,
- * or the device returned QUEUE_FULL and can accept no more
- * commands.
- * Notes: This could be called either from an interrupt context or a
- * normal process context.
+ * Context: This could be called either from an interrupt context or a normal
+ * process context.
*/
void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
{
@@ -257,7 +249,8 @@
req = blk_get_request(sdev->request_queue,
data_direction == DMA_TO_DEVICE ?
- REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, BLK_MQ_REQ_PREEMPT);
+ REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN,
+ rq_flags & RQF_PM ? BLK_MQ_REQ_PM : 0);
if (IS_ERR(req))
return ret;
rq = scsi_req(req);
@@ -302,39 +295,20 @@
EXPORT_SYMBOL(__scsi_execute);
/*
- * Function: scsi_init_cmd_errh()
- *
- * Purpose: Initialize cmd fields related to error handling.
- *
- * Arguments: cmd - command that is ready to be queued.
- *
- * Notes: This function has the job of initializing a number of
- * fields related to error handling. Typically this will
- * be called once for each command, as required.
- */
-static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
-{
- scsi_set_resid(cmd, 0);
- memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
- if (cmd->cmd_len == 0)
- cmd->cmd_len = scsi_command_size(cmd->cmnd);
-}
-
-/*
- * Decrement the host_busy counter and wake up the error handler if necessary.
- * Avoid as follows that the error handler is not woken up if shost->host_busy
- * == shost->host_failed: use call_rcu() in scsi_eh_scmd_add() in combination
- * with an RCU read lock in this function to ensure that this function in its
- * entirety either finishes before scsi_eh_scmd_add() increases the
+ * Wake up the error handler if necessary. Avoid as follows that the error
+ * handler is not woken up if host in-flight requests number ==
+ * shost->host_failed: use call_rcu() in scsi_eh_scmd_add() in combination
+ * with an RCU read lock in this function to ensure that this function in
+ * its entirety either finishes before scsi_eh_scmd_add() increases the
* host_failed counter or that it notices the shost state change made by
* scsi_eh_scmd_add().
*/
-static void scsi_dec_host_busy(struct Scsi_Host *shost)
+static void scsi_dec_host_busy(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
{
unsigned long flags;
rcu_read_lock();
- atomic_dec(&shost->host_busy);
+ __clear_bit(SCMD_STATE_INFLIGHT, &cmd->state);
if (unlikely(scsi_host_in_recovery(shost))) {
spin_lock_irqsave(shost->host_lock, flags);
if (shost->host_failed || shost->host_eh_scheduled)
@@ -344,12 +318,12 @@
rcu_read_unlock();
}
-void scsi_device_unbusy(struct scsi_device *sdev)
+void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd)
{
struct Scsi_Host *shost = sdev->host;
struct scsi_target *starget = scsi_target(sdev);
- scsi_dec_host_busy(shost);
+ scsi_dec_host_busy(shost, cmd);
if (starget->can_queue > 0)
atomic_dec(&starget->target_busy);
@@ -401,7 +375,7 @@
spin_unlock_irqrestore(shost->host_lock, flags);
scsi_kick_queue(sdev->request_queue);
spin_lock_irqsave(shost->host_lock, flags);
-
+
scsi_device_put(sdev);
}
out:
@@ -430,9 +404,6 @@
static inline bool scsi_host_is_busy(struct Scsi_Host *shost)
{
- if (shost->can_queue > 0 &&
- atomic_read(&shost->host_busy) >= shost->can_queue)
- return true;
if (atomic_read(&shost->host_blocked) > 0)
return true;
if (shost->host_self_blocked)
@@ -499,17 +470,11 @@
spin_unlock_irqrestore(shost->host_lock, flags);
}
-/*
- * Function: scsi_run_queue()
+/**
+ * scsi_run_queue - Select a proper request queue to serve next.
+ * @q: last request's queue
*
- * Purpose: Select a proper request queue to serve next
- *
- * Arguments: q - last request's queue
- *
- * Returns: Nothing
- *
- * Notes: The previous command was completely finished, start
- * a new one if possible.
+ * The previous command was completely finished, start a new one if possible.
*/
static void scsi_run_queue(struct request_queue *q)
{
@@ -551,7 +516,7 @@
}
}
-static void scsi_free_sgtables(struct scsi_cmnd *cmd)
+void scsi_free_sgtables(struct scsi_cmnd *cmd)
{
if (cmd->sdb.table.nents)
sg_free_table_chained(&cmd->sdb.table,
@@ -560,21 +525,38 @@
sg_free_table_chained(&cmd->prot_sdb->table,
SCSI_INLINE_PROT_SG_CNT);
}
+EXPORT_SYMBOL_GPL(scsi_free_sgtables);
static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)
{
scsi_free_sgtables(cmd);
scsi_uninit_cmd(cmd);
- scsi_del_cmd_from_list(cmd);
}
static void scsi_run_queue_async(struct scsi_device *sdev)
{
if (scsi_target(sdev)->single_lun ||
- !list_empty(&sdev->host->starved_list))
+ !list_empty(&sdev->host->starved_list)) {
kblockd_schedule_work(&sdev->requeue_work);
- else
- blk_mq_run_hw_queues(sdev->request_queue, true);
+ } else {
+ /*
+ * smp_mb() present in sbitmap_queue_clear() or implied in
+ * .end_io is for ordering writing .device_busy in
+ * scsi_device_unbusy() and reading sdev->restarts.
+ */
+ int old = atomic_read(&sdev->restarts);
+
+ /*
+ * ->restarts has to be kept as non-zero if new budget
+ * contention occurs.
+ *
+ * No need to run queue when either another re-run
+ * queue wins in updating ->restarts or a new budget
+ * contention occurs.
+ */
+ if (old && atomic_cmpxchg(&sdev->restarts, old, 0) == old)
+ blk_mq_run_hw_queues(sdev->request_queue, true);
+ }
}
/* Returns false when no more bytes to process, true if there are more */
@@ -674,6 +656,23 @@
scsi_mq_requeue_cmd(cmd);
}
+static bool scsi_cmd_runtime_exceeced(struct scsi_cmnd *cmd)
+{
+ struct request *req = cmd->request;
+ unsigned long wait_for;
+
+ if (cmd->allowed == SCSI_CMD_RETRIES_NO_LIMIT)
+ return false;
+
+ wait_for = (cmd->allowed + 1) * req->timeout;
+ if (time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
+ scmd_printk(KERN_ERR, cmd, "timing out command, waited %lus\n",
+ wait_for/HZ);
+ return true;
+ }
+ return false;
+}
+
/* Helper for scsi_io_completion() when special action required. */
static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result)
{
@@ -682,7 +681,6 @@
int level = 0;
enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
ACTION_DELAYED_RETRY} action;
- unsigned long wait_for = (cmd->allowed + 1) * req->timeout;
struct scsi_sense_hdr sshdr;
bool sense_valid;
bool sense_current = true; /* false implies "deferred sense" */
@@ -781,6 +779,15 @@
/* See SSC3rXX or current. */
action = ACTION_FAIL;
break;
+ case DATA_PROTECT:
+ action = ACTION_FAIL;
+ if ((sshdr.asc == 0x0C && sshdr.ascq == 0x12) ||
+ (sshdr.asc == 0x55 &&
+ (sshdr.ascq == 0x0E || sshdr.ascq == 0x0F))) {
+ /* Insufficient zone resources */
+ blk_stat = BLK_STS_ZONE_OPEN_RESOURCE;
+ }
+ break;
default:
action = ACTION_FAIL;
break;
@@ -788,8 +795,7 @@
} else
action = ACTION_FAIL;
- if (action != ACTION_FAIL &&
- time_before(cmd->jiffies_at_alloc + wait_for, jiffies))
+ if (action != ACTION_FAIL && scsi_cmd_runtime_exceeced(cmd))
action = ACTION_FAIL;
switch (action) {
@@ -818,7 +824,7 @@
}
if (!scsi_end_request(req, blk_stat, blk_rq_err_bytes(req)))
return;
- /*FALLTHRU*/
+ fallthrough;
case ACTION_REPREP:
scsi_io_completion_reprep(cmd, q);
break;
@@ -906,34 +912,27 @@
return result;
}
-/*
- * Function: scsi_io_completion()
+/**
+ * scsi_io_completion - Completion processing for SCSI commands.
+ * @cmd: command that is finished.
+ * @good_bytes: number of processed bytes.
*
- * Purpose: Completion processing for block device I/O requests.
+ * We will finish off the specified number of sectors. If we are done, the
+ * command block will be released and the queue function will be goosed. If we
+ * are not done then we have to figure out what to do next:
*
- * Arguments: cmd - command that is finished.
+ * a) We can call scsi_io_completion_reprep(). The request will be
+ * unprepared and put back on the queue. Then a new command will
+ * be created for it. This should be used if we made forward
+ * progress, or if we want to switch from READ(10) to READ(6) for
+ * example.
*
- * Lock status: Assumed that no lock is held upon entry.
+ * b) We can call scsi_io_completion_action(). The request will be
+ * put back on the queue and retried using the same command as
+ * before, possibly after a delay.
*
- * Returns: Nothing
- *
- * Notes: We will finish off the specified number of sectors. If we
- * are done, the command block will be released and the queue
- * function will be goosed. If we are not done then we have to
- * figure out what to do next:
- *
- * a) We can call scsi_requeue_command(). The request
- * will be unprepared and put back on the queue. Then
- * a new command will be created for it. This should
- * be used if we made forward progress, or if we want
- * to switch from READ(10) to READ(6) for example.
- *
- * b) We can call __scsi_queue_insert(). The request will
- * be put back on the queue and retried using the same
- * command as before, possibly after a delay.
- *
- * c) We can call scsi_end_request() with blk_stat other than
- * BLK_STS_OK, to fail the remainder of the request.
+ * c) We can call scsi_end_request() with blk_stat other than
+ * BLK_STS_OK, to fail the remainder of the request.
*/
void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
{
@@ -961,8 +960,7 @@
blk_rq_sectors(req), good_bytes));
/*
- * Next deal with any sectors which we were able to correctly
- * handle. Failed, zero length commands always need to drop down
+ * Failed, zero length commands always need to drop down
* to retry code. Fast path should return in this block.
*/
if (likely(blk_rq_bytes(req) > 0 || blk_stat == BLK_STS_OK)) {
@@ -988,56 +986,81 @@
scsi_io_completion_action(cmd, result);
}
-static blk_status_t scsi_init_sgtable(struct request *req,
- struct scsi_data_buffer *sdb)
+static inline bool scsi_cmd_needs_dma_drain(struct scsi_device *sdev,
+ struct request *rq)
{
+ return sdev->dma_drain_len && blk_rq_is_passthrough(rq) &&
+ !op_is_write(req_op(rq)) &&
+ sdev->host->hostt->dma_need_drain(rq);
+}
+
+/**
+ * scsi_alloc_sgtables - allocate S/G tables for a command
+ * @cmd: command descriptor we wish to initialize
+ *
+ * Returns:
+ * * BLK_STS_OK - on success
+ * * BLK_STS_RESOURCE - if the failure is retryable
+ * * BLK_STS_IOERR - if the failure is fatal
+ */
+blk_status_t scsi_alloc_sgtables(struct scsi_cmnd *cmd)
+{
+ struct scsi_device *sdev = cmd->device;
+ struct request *rq = cmd->request;
+ unsigned short nr_segs = blk_rq_nr_phys_segments(rq);
+ struct scatterlist *last_sg = NULL;
+ blk_status_t ret;
+ bool need_drain = scsi_cmd_needs_dma_drain(sdev, rq);
int count;
+ if (WARN_ON_ONCE(!nr_segs))
+ return BLK_STS_IOERR;
+
+ /*
+ * Make sure there is space for the drain. The driver must adjust
+ * max_hw_segments to be prepared for this.
+ */
+ if (need_drain)
+ nr_segs++;
+
/*
* If sg table allocation fails, requeue request later.
*/
- if (unlikely(sg_alloc_table_chained(&sdb->table,
- blk_rq_nr_phys_segments(req), sdb->table.sgl,
- SCSI_INLINE_SG_CNT)))
+ if (unlikely(sg_alloc_table_chained(&cmd->sdb.table, nr_segs,
+ cmd->sdb.table.sgl, SCSI_INLINE_SG_CNT)))
return BLK_STS_RESOURCE;
- /*
+ /*
* Next, walk the list, and fill in the addresses and sizes of
* each segment.
*/
- count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
- BUG_ON(count > sdb->table.nents);
- sdb->table.nents = count;
- sdb->length = blk_rq_payload_bytes(req);
- return BLK_STS_OK;
-}
+ count = __blk_rq_map_sg(rq->q, rq, cmd->sdb.table.sgl, &last_sg);
-/*
- * Function: scsi_init_io()
- *
- * Purpose: SCSI I/O initialize function.
- *
- * Arguments: cmd - Command descriptor we wish to initialize
- *
- * Returns: BLK_STS_OK on success
- * BLK_STS_RESOURCE if the failure is retryable
- * BLK_STS_IOERR if the failure is fatal
- */
-blk_status_t scsi_init_io(struct scsi_cmnd *cmd)
-{
- struct request *rq = cmd->request;
- blk_status_t ret;
+ if (blk_rq_bytes(rq) & rq->q->dma_pad_mask) {
+ unsigned int pad_len =
+ (rq->q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;
- if (WARN_ON_ONCE(!blk_rq_nr_phys_segments(rq)))
- return BLK_STS_IOERR;
+ last_sg->length += pad_len;
+ cmd->extra_len += pad_len;
+ }
- ret = scsi_init_sgtable(rq, &cmd->sdb);
- if (ret)
- return ret;
+ if (need_drain) {
+ sg_unmark_end(last_sg);
+ last_sg = sg_next(last_sg);
+ sg_set_buf(last_sg, sdev->dma_drain_buf, sdev->dma_drain_len);
+ sg_mark_end(last_sg);
+
+ cmd->extra_len += sdev->dma_drain_len;
+ count++;
+ }
+
+ BUG_ON(count > cmd->sdb.table.nents);
+ cmd->sdb.table.nents = count;
+ cmd->sdb.length = blk_rq_payload_bytes(rq);
if (blk_integrity_rq(rq)) {
struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
- int ivecs, count;
+ int ivecs;
if (WARN_ON_ONCE(!prot_sdb)) {
/*
@@ -1072,7 +1095,7 @@
scsi_free_sgtables(cmd);
return ret;
}
-EXPORT_SYMBOL(scsi_init_io);
+EXPORT_SYMBOL(scsi_alloc_sgtables);
/**
* scsi_initialize_rq - initialize struct scsi_cmnd partially
@@ -1107,36 +1130,7 @@
}
}
-/* Add a command to the list used by the aacraid and dpt_i2o drivers */
-void scsi_add_cmd_to_list(struct scsi_cmnd *cmd)
-{
- struct scsi_device *sdev = cmd->device;
- struct Scsi_Host *shost = sdev->host;
- unsigned long flags;
-
- if (shost->use_cmd_list) {
- spin_lock_irqsave(&sdev->list_lock, flags);
- list_add_tail(&cmd->list, &sdev->cmd_list);
- spin_unlock_irqrestore(&sdev->list_lock, flags);
- }
-}
-
-/* Remove a command from the list used by the aacraid and dpt_i2o drivers */
-void scsi_del_cmd_from_list(struct scsi_cmnd *cmd)
-{
- struct scsi_device *sdev = cmd->device;
- struct Scsi_Host *shost = sdev->host;
- unsigned long flags;
-
- if (shost->use_cmd_list) {
- spin_lock_irqsave(&sdev->list_lock, flags);
- BUG_ON(list_empty(&cmd->list));
- list_del_init(&cmd->list);
- spin_unlock_irqrestore(&sdev->list_lock, flags);
- }
-}
-
-/* Called after a request has been started. */
+/* Called before a request is prepared. See also scsi_mq_prep_fn(). */
void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd)
{
void *buf = cmd->sense_buffer;
@@ -1144,7 +1138,8 @@
struct request *rq = blk_mq_rq_from_pdu(cmd);
unsigned int flags = cmd->flags & SCMD_PRESERVED_FLAGS;
unsigned long jiffies_at_alloc;
- int retries;
+ int retries, to_clear;
+ bool in_flight;
if (!blk_rq_is_scsi(rq) && !(flags & SCMD_INITIALIZED)) {
flags |= SCMD_INITIALIZED;
@@ -1153,9 +1148,16 @@
jiffies_at_alloc = cmd->jiffies_at_alloc;
retries = cmd->retries;
- /* zero out the cmd, except for the embedded scsi_request */
- memset((char *)cmd + sizeof(cmd->req), 0,
- sizeof(*cmd) - sizeof(cmd->req) + dev->host->hostt->cmd_size);
+ in_flight = test_bit(SCMD_STATE_INFLIGHT, &cmd->state);
+ /*
+ * Zero out the cmd, except for the embedded scsi_request. Only clear
+ * the driver-private command data if the LLD does not supply a
+ * function to initialize that data.
+ */
+ to_clear = sizeof(*cmd) - sizeof(cmd->req);
+ if (!dev->host->hostt->init_cmd_priv)
+ to_clear += dev->host->hostt->cmd_size;
+ memset((char *)cmd + sizeof(cmd->req), 0, to_clear);
cmd->device = dev;
cmd->sense_buffer = buf;
@@ -1164,8 +1166,9 @@
INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);
cmd->jiffies_at_alloc = jiffies_at_alloc;
cmd->retries = retries;
+ if (in_flight)
+ __set_bit(SCMD_STATE_INFLIGHT, &cmd->state);
- scsi_add_cmd_to_list(cmd);
}
static blk_status_t scsi_setup_scsi_cmnd(struct scsi_device *sdev,
@@ -1180,7 +1183,7 @@
* submit a request without an attached bio.
*/
if (req->bio) {
- blk_status_t ret = scsi_init_io(cmd);
+ blk_status_t ret = scsi_alloc_sgtables(cmd);
if (unlikely(ret != BLK_STS_OK))
return ret;
} else {
@@ -1196,54 +1199,12 @@
return BLK_STS_OK;
}
-/*
- * Setup a normal block command. These are simple request from filesystems
- * that still need to be translated to SCSI CDBs from the ULD.
- */
-static blk_status_t scsi_setup_fs_cmnd(struct scsi_device *sdev,
- struct request *req)
-{
- struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
-
- if (unlikely(sdev->handler && sdev->handler->prep_fn)) {
- blk_status_t ret = sdev->handler->prep_fn(sdev, req);
- if (ret != BLK_STS_OK)
- return ret;
- }
-
- cmd->cmnd = scsi_req(req)->cmd = scsi_req(req)->__cmd;
- memset(cmd->cmnd, 0, BLK_MAX_CDB);
- return scsi_cmd_to_driver(cmd)->init_command(cmd);
-}
-
-static blk_status_t scsi_setup_cmnd(struct scsi_device *sdev,
- struct request *req)
-{
- struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
- blk_status_t ret;
-
- if (!blk_rq_bytes(req))
- cmd->sc_data_direction = DMA_NONE;
- else if (rq_data_dir(req) == WRITE)
- cmd->sc_data_direction = DMA_TO_DEVICE;
- else
- cmd->sc_data_direction = DMA_FROM_DEVICE;
-
- if (blk_rq_is_scsi(req))
- ret = scsi_setup_scsi_cmnd(sdev, req);
- else
- ret = scsi_setup_fs_cmnd(sdev, req);
-
- if (ret != BLK_STS_OK)
- scsi_free_sgtables(cmd);
-
- return ret;
-}
-
static blk_status_t
-scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
+scsi_device_state_check(struct scsi_device *sdev, struct request *req)
{
switch (sdev->sdev_state) {
+ case SDEV_CREATED:
+ return BLK_STS_OK;
case SDEV_OFFLINE:
case SDEV_TRANSPORT_OFFLINE:
/*
@@ -1251,8 +1212,11 @@
* commands. The device must be brought online
* before trying any recovery commands.
*/
- sdev_printk(KERN_ERR, sdev,
- "rejecting I/O to offline device\n");
+ if (!sdev->offline_already) {
+ sdev->offline_already = true;
+ sdev_printk(KERN_ERR, sdev,
+ "rejecting I/O to offline device\n");
+ }
return BLK_STS_IOERR;
case SDEV_DEL:
/*
@@ -1267,18 +1231,18 @@
return BLK_STS_RESOURCE;
case SDEV_QUIESCE:
/*
- * If the devices is blocked we defer normal commands.
+ * If the device is blocked we only accept power management
+ * commands.
*/
- if (req && !(req->rq_flags & RQF_PREEMPT))
+ if (req && WARN_ON_ONCE(!(req->rq_flags & RQF_PM)))
return BLK_STS_RESOURCE;
return BLK_STS_OK;
default:
/*
* For any other not fully online state we only allow
- * special commands. In particular any user initiated
- * command is not allowed.
+ * power management commands.
*/
- if (req && !(req->rq_flags & RQF_PREEMPT))
+ if (req && !(req->rq_flags & RQF_PM))
return BLK_STS_IOERR;
return BLK_STS_OK;
}
@@ -1379,16 +1343,14 @@
*/
static inline int scsi_host_queue_ready(struct request_queue *q,
struct Scsi_Host *shost,
- struct scsi_device *sdev)
+ struct scsi_device *sdev,
+ struct scsi_cmnd *cmd)
{
- unsigned int busy;
-
if (scsi_host_in_recovery(shost))
return 0;
- busy = atomic_inc_return(&shost->host_busy) - 1;
if (atomic_read(&shost->host_blocked) > 0) {
- if (busy)
+ if (scsi_host_busy(shost) > 0)
goto starved;
/*
@@ -1402,8 +1364,6 @@
"unblocking host at zero depth\n"));
}
- if (shost->can_queue > 0 && busy >= shost->can_queue)
- goto starved;
if (shost->host_self_blocked)
goto starved;
@@ -1415,6 +1375,8 @@
spin_unlock_irq(shost->host_lock);
}
+ __set_bit(SCMD_STATE_INFLIGHT, &cmd->state);
+
return 1;
starved:
@@ -1423,7 +1385,7 @@
list_add_tail(&sdev->starved_entry, &shost->starved_list);
spin_unlock_irq(shost->host_lock);
out_dec:
- scsi_dec_host_busy(shost);
+ scsi_dec_host_busy(shost, cmd);
return 0;
}
@@ -1464,7 +1426,6 @@
static void scsi_softirq_done(struct request *rq)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
- unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
int disposition;
INIT_LIST_HEAD(&cmd->eh_entry);
@@ -1474,29 +1435,24 @@
atomic_inc(&cmd->device->ioerr_cnt);
disposition = scsi_decide_disposition(cmd);
- if (disposition != SUCCESS &&
- time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
- scmd_printk(KERN_ERR, cmd,
- "timing out command, waited %lus\n",
- wait_for/HZ);
+ if (disposition != SUCCESS && scsi_cmd_runtime_exceeced(cmd))
disposition = SUCCESS;
- }
scsi_log_completion(cmd, disposition);
switch (disposition) {
- case SUCCESS:
- scsi_finish_command(cmd);
- break;
- case NEEDS_RETRY:
- scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
- break;
- case ADD_TO_MLQUEUE:
- scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
- break;
- default:
- scsi_eh_scmd_add(cmd);
- break;
+ case SUCCESS:
+ scsi_finish_command(cmd);
+ break;
+ case NEEDS_RETRY:
+ scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
+ break;
+ case ADD_TO_MLQUEUE:
+ scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
+ break;
+ default:
+ scsi_eh_scmd_add(cmd);
+ break;
}
}
@@ -1588,7 +1544,7 @@
sizeof(struct scatterlist);
}
-static blk_status_t scsi_mq_prep_fn(struct request *req)
+static blk_status_t scsi_prepare_cmd(struct request *req)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
struct scsi_device *sdev = req->q->queuedata;
@@ -1600,6 +1556,10 @@
cmd->request = req;
cmd->tag = req->tag;
cmd->prot_op = SCSI_PROT_NORMAL;
+ if (blk_rq_bytes(req))
+ cmd->sc_data_direction = rq_dma_dir(req);
+ else
+ cmd->sc_data_direction = DMA_NONE;
sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;
cmd->sdb.table.sgl = sg;
@@ -1611,45 +1571,69 @@
(struct scatterlist *)(cmd->prot_sdb + 1);
}
- blk_mq_start_request(req);
+ /*
+ * Special handling for passthrough commands, which don't go to the ULP
+ * at all:
+ */
+ if (blk_rq_is_scsi(req))
+ return scsi_setup_scsi_cmnd(sdev, req);
- return scsi_setup_cmnd(sdev, req);
+ if (sdev->handler && sdev->handler->prep_fn) {
+ blk_status_t ret = sdev->handler->prep_fn(sdev, req);
+
+ if (ret != BLK_STS_OK)
+ return ret;
+ }
+
+ cmd->cmnd = scsi_req(req)->cmd = scsi_req(req)->__cmd;
+ memset(cmd->cmnd, 0, BLK_MAX_CDB);
+ return scsi_cmd_to_driver(cmd)->init_command(cmd);
}
static void scsi_mq_done(struct scsi_cmnd *cmd)
{
+ if (unlikely(blk_should_fake_timeout(cmd->request->q)))
+ return;
if (unlikely(test_and_set_bit(SCMD_STATE_COMPLETE, &cmd->state)))
return;
trace_scsi_dispatch_cmd_done(cmd);
-
- /*
- * If the block layer didn't complete the request due to a timeout
- * injection, scsi must clear its internal completed state so that the
- * timeout handler will see it needs to escalate its own error
- * recovery.
- */
- if (unlikely(!blk_mq_complete_request(cmd->request)))
- clear_bit(SCMD_STATE_COMPLETE, &cmd->state);
+ blk_mq_complete_request(cmd->request);
}
-static void scsi_mq_put_budget(struct blk_mq_hw_ctx *hctx)
+static void scsi_mq_put_budget(struct request_queue *q)
{
- struct request_queue *q = hctx->queue;
struct scsi_device *sdev = q->queuedata;
atomic_dec(&sdev->device_busy);
}
-static bool scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)
+static bool scsi_mq_get_budget(struct request_queue *q)
{
- struct request_queue *q = hctx->queue;
struct scsi_device *sdev = q->queuedata;
if (scsi_dev_queue_ready(q, sdev))
return true;
- if (atomic_read(&sdev->device_busy) == 0 && !scsi_device_blocked(sdev))
- blk_mq_delay_run_hw_queue(hctx, SCSI_QUEUE_DELAY);
+ atomic_inc(&sdev->restarts);
+
+ /*
+ * Orders atomic_inc(&sdev->restarts) and atomic_read(&sdev->device_busy).
+ * .restarts must be incremented before .device_busy is read because the
+ * code in scsi_run_queue_async() depends on the order of these operations.
+ */
+ smp_mb__after_atomic();
+
+ /*
+ * If all in-flight requests originated from this LUN are completed
+ * before reading .device_busy, sdev->device_busy will be observed as
+ * zero, then blk_mq_delay_run_hw_queues() will dispatch this request
+ * soon. Otherwise, completion of one of these requests will observe
+ * the .restarts flag, and the request queue will be run for handling
+ * this request, see scsi_end_request().
+ */
+ if (unlikely(atomic_read(&sdev->device_busy) == 0 &&
+ !scsi_device_blocked(sdev)))
+ blk_mq_delay_run_hw_queues(sdev->request_queue, SCSI_QUEUE_DELAY);
return false;
}
@@ -1669,7 +1653,7 @@
* commands.
*/
if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
- ret = scsi_prep_state_check(sdev, req);
+ ret = scsi_device_state_check(sdev, req);
if (ret != BLK_STS_OK)
goto out_put_budget;
}
@@ -1677,17 +1661,16 @@
ret = BLK_STS_RESOURCE;
if (!scsi_target_queue_ready(shost, sdev))
goto out_put_budget;
- if (!scsi_host_queue_ready(q, shost, sdev))
+ if (!scsi_host_queue_ready(q, shost, sdev, cmd))
goto out_dec_target_busy;
if (!(req->rq_flags & RQF_DONTPREP)) {
- ret = scsi_mq_prep_fn(req);
+ ret = scsi_prepare_cmd(req);
if (ret != BLK_STS_OK)
goto out_dec_host_busy;
req->rq_flags |= RQF_DONTPREP;
} else {
clear_bit(SCMD_STATE_COMPLETE, &cmd->state);
- blk_mq_start_request(req);
}
cmd->flags &= SCMD_PRESERVED_FLAGS;
@@ -1696,9 +1679,11 @@
if (bd->last)
cmd->flags |= SCMD_LAST;
- scsi_init_cmd_errh(cmd);
+ scsi_set_resid(cmd, 0);
+ memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
cmd->scsi_done = scsi_mq_done;
+ blk_mq_start_request(req);
reason = scsi_dispatch_cmd(cmd);
if (reason) {
scsi_set_blocked(cmd, reason);
@@ -1709,18 +1694,18 @@
return BLK_STS_OK;
out_dec_host_busy:
- scsi_dec_host_busy(shost);
+ scsi_dec_host_busy(shost, cmd);
out_dec_target_busy:
if (scsi_target(sdev)->can_queue > 0)
atomic_dec(&scsi_target(sdev)->target_busy);
out_put_budget:
- scsi_mq_put_budget(hctx);
+ scsi_mq_put_budget(q);
switch (ret) {
case BLK_STS_OK:
break;
case BLK_STS_RESOURCE:
- if (atomic_read(&sdev->device_busy) ||
- scsi_device_blocked(sdev))
+ case BLK_STS_ZONE_RESOURCE:
+ if (scsi_device_blocked(sdev))
ret = BLK_STS_DEV_RESOURCE;
break;
default:
@@ -1756,6 +1741,7 @@
const bool unchecked_isa_dma = shost->unchecked_isa_dma;
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
struct scatterlist *sg;
+ int ret = 0;
if (unchecked_isa_dma)
cmd->flags |= SCMD_UNCHECKED_ISA_DMA;
@@ -1771,14 +1757,24 @@
cmd->prot_sdb = (void *)sg + scsi_mq_inline_sgl_size(shost);
}
- return 0;
+ if (shost->hostt->init_cmd_priv) {
+ ret = shost->hostt->init_cmd_priv(shost, cmd);
+ if (ret < 0)
+ scsi_free_sense_buffer(unchecked_isa_dma,
+ cmd->sense_buffer);
+ }
+
+ return ret;
}
static void scsi_mq_exit_request(struct blk_mq_tag_set *set, struct request *rq,
unsigned int hctx_idx)
{
+ struct Scsi_Host *shost = set->driver_data;
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
+ if (shost->hostt->exit_cmd_priv)
+ shost->hostt->exit_cmd_priv(shost, cmd);
scsi_free_sense_buffer(cmd->flags & SCMD_UNCHECKED_ISA_DMA,
cmd->sense_buffer);
}
@@ -1895,6 +1891,7 @@
int scsi_mq_setup_tags(struct Scsi_Host *shost)
{
unsigned int cmd_size, sgl_size;
+ struct blk_mq_tag_set *tag_set = &shost->tag_set;
sgl_size = max_t(unsigned int, sizeof(struct scatterlist),
scsi_mq_inline_sgl_size(shost));
@@ -1903,21 +1900,23 @@
cmd_size += sizeof(struct scsi_data_buffer) +
sizeof(struct scatterlist) * SCSI_INLINE_PROT_SG_CNT;
- memset(&shost->tag_set, 0, sizeof(shost->tag_set));
+ memset(tag_set, 0, sizeof(*tag_set));
if (shost->hostt->commit_rqs)
- shost->tag_set.ops = &scsi_mq_ops;
+ tag_set->ops = &scsi_mq_ops;
else
- shost->tag_set.ops = &scsi_mq_ops_no_commit;
- shost->tag_set.nr_hw_queues = shost->nr_hw_queues ? : 1;
- shost->tag_set.queue_depth = shost->can_queue;
- shost->tag_set.cmd_size = cmd_size;
- shost->tag_set.numa_node = NUMA_NO_NODE;
- shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
- shost->tag_set.flags |=
+ tag_set->ops = &scsi_mq_ops_no_commit;
+ tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1;
+ tag_set->queue_depth = shost->can_queue;
+ tag_set->cmd_size = cmd_size;
+ tag_set->numa_node = NUMA_NO_NODE;
+ tag_set->flags = BLK_MQ_F_SHOULD_MERGE;
+ tag_set->flags |=
BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy);
- shost->tag_set.driver_data = shost;
+ tag_set->driver_data = shost;
+ if (shost->host_tagset)
+ tag_set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
- return blk_mq_alloc_tag_set(&shost->tag_set);
+ return blk_mq_alloc_tag_set(tag_set);
}
void scsi_mq_destroy_tags(struct Scsi_Host *shost)
@@ -1944,23 +1943,14 @@
return sdev;
}
-EXPORT_SYMBOL_GPL(scsi_device_from_queue);
-/*
- * Function: scsi_block_requests()
+/**
+ * scsi_block_requests - Utility function used by low-level drivers to prevent
+ * further commands from being queued to the device.
+ * @shost: host in question
*
- * Purpose: Utility function used by low-level drivers to prevent further
- * commands from being queued to the device.
- *
- * Arguments: shost - Host in question
- *
- * Returns: Nothing
- *
- * Lock status: No locks are assumed held.
- *
- * Notes: There is no timer nor any other means by which the requests
- * get unblocked other than the low-level driver calling
- * scsi_unblock_requests().
+ * There is no timer nor any other means by which the requests get unblocked
+ * other than the low-level driver calling scsi_unblock_requests().
*/
void scsi_block_requests(struct Scsi_Host *shost)
{
@@ -1968,25 +1958,15 @@
}
EXPORT_SYMBOL(scsi_block_requests);
-/*
- * Function: scsi_unblock_requests()
+/**
+ * scsi_unblock_requests - Utility function used by low-level drivers to allow
+ * further commands to be queued to the device.
+ * @shost: host in question
*
- * Purpose: Utility function used by low-level drivers to allow further
- * commands from being queued to the device.
- *
- * Arguments: shost - Host in question
- *
- * Returns: Nothing
- *
- * Lock status: No locks are assumed held.
- *
- * Notes: There is no timer nor any other means by which the requests
- * get unblocked other than the low-level driver calling
- * scsi_unblock_requests().
- *
- * This is done as an API function so that changes to the
- * internals of the scsi mid-layer won't require wholesale
- * changes to drivers that use this feature.
+ * There is no timer nor any other means by which the requests get unblocked
+ * other than the low-level driver calling scsi_unblock_requests(). This is done
+ * as an API function so that changes to the internals of the scsi mid-layer
+ * won't require wholesale changes to drivers that use this feature.
*/
void scsi_unblock_requests(struct Scsi_Host *shost)
{
@@ -1995,24 +1975,10 @@
}
EXPORT_SYMBOL(scsi_unblock_requests);
-int __init scsi_init_queue(void)
-{
- scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
- sizeof(struct scsi_data_buffer),
- 0, 0, NULL);
- if (!scsi_sdb_cache) {
- printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
- return -ENOMEM;
- }
-
- return 0;
-}
-
void scsi_exit_queue(void)
{
kmem_cache_destroy(scsi_sense_cache);
kmem_cache_destroy(scsi_sense_isadma_cache);
- kmem_cache_destroy(scsi_sdb_cache);
}
/**
@@ -2079,7 +2045,6 @@
real_buffer[1] = data->medium_type;
real_buffer[2] = data->device_specific;
real_buffer[3] = data->block_descriptor_length;
-
cmd[0] = MODE_SELECT;
cmd[4] = len;
@@ -2105,9 +2070,7 @@
* @sshdr: place to put sense data (or NULL if no sense to be collected).
* must be SCSI_SENSE_BUFFERSIZE big.
*
- * Returns zero if unsuccessful, or the header offset (either 4
- * or 8 depending on whether a six or ten byte command was
- * issued) if successful.
+ * Returns zero if successful, or a negative error number on failure
*/
int
scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
@@ -2122,6 +2085,8 @@
memset(data, 0, sizeof(*data));
memset(&cmd[0], 0, 12);
+
+ dbd = sdev->set_dbd_for_ms ? 8 : dbd;
cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */
cmd[2] = modepage;
@@ -2152,6 +2117,8 @@
result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
sshdr, timeout, retries, NULL);
+ if (result < 0)
+ return result;
/* This code looks awful: what it's doing is making sure an
* ILLEGAL REQUEST sense return identifies the actual command
@@ -2163,7 +2130,7 @@
if (scsi_sense_valid(sshdr)) {
if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
(sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
- /*
+ /*
* Invalid command operation code
*/
sdev->use_10_for_ms = 0;
@@ -2172,7 +2139,7 @@
}
}
- if(scsi_status_is_good(result)) {
+ if (scsi_status_is_good(result)) {
if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
(modepage == 6 || modepage == 8))) {
/* Initio breakage? */
@@ -2182,7 +2149,7 @@
data->device_specific = 0;
data->longlba = 0;
data->block_descriptor_length = 0;
- } else if(use_10_for_ms) {
+ } else if (use_10_for_ms) {
data->length = buffer[0]*256 + buffer[1] + 2;
data->medium_type = buffer[2];
data->device_specific = buffer[3];
@@ -2196,13 +2163,15 @@
data->block_descriptor_length = buffer[3];
}
data->header_length = header_length;
+ result = 0;
} else if ((status_byte(result) == CHECK_CONDITION) &&
scsi_sense_valid(sshdr) &&
sshdr->sense_key == UNIT_ATTENTION && retry_count) {
retry_count--;
goto retry;
}
-
+ if (result > 0)
+ result = -EIO;
return result;
}
EXPORT_SYMBOL(scsi_mode_sense);
@@ -2265,7 +2234,7 @@
goto illegal;
}
break;
-
+
case SDEV_RUNNING:
switch (oldstate) {
case SDEV_CREATED:
@@ -2307,6 +2276,7 @@
switch (oldstate) {
case SDEV_RUNNING:
case SDEV_CREATED_BLOCK:
+ case SDEV_QUIESCE:
case SDEV_OFFLINE:
break;
default:
@@ -2352,6 +2322,7 @@
break;
}
+ sdev->offline_already = false;
sdev->sdev_state = state;
return 0;
@@ -2541,15 +2512,13 @@
EXPORT_SYMBOL_GPL(sdev_evt_send_simple);
/**
- * scsi_device_quiesce - Block user issued commands.
+ * scsi_device_quiesce - Block all commands except power management.
* @sdev: scsi device to quiesce.
*
* This works by trying to transition to the SDEV_QUIESCE state
* (which must be a legal transition). When the device is in this
- * state, only special requests will be accepted, all others will
- * be deferred. Since special requests may also be requeued requests,
- * a successful return doesn't guarantee the device will be
- * totally quiescent.
+ * state, only power management requests will be accepted, all others will
+ * be deferred.
*
* Must be called with user context, may sleep.
*
@@ -2611,12 +2580,12 @@
* device deleted during suspend)
*/
mutex_lock(&sdev->state_mutex);
+ if (sdev->sdev_state == SDEV_QUIESCE)
+ scsi_device_set_state(sdev, SDEV_RUNNING);
if (sdev->quiesced_by) {
sdev->quiesced_by = NULL;
blk_clear_pm_only(sdev->request_queue);
}
- if (sdev->sdev_state == SDEV_QUIESCE)
- scsi_device_set_state(sdev, SDEV_RUNNING);
mutex_unlock(&sdev->state_mutex);
}
EXPORT_SYMBOL(scsi_device_resume);
@@ -2674,10 +2643,10 @@
return err;
}
- /*
+ /*
* The device has transitioned to SDEV_BLOCK. Stop the
* block layer from calling the midlayer with this device's
- * request queue.
+ * request queue.
*/
blk_mq_quiesce_queue_nowait(q);
return 0;
@@ -2712,7 +2681,7 @@
return err;
}
-
+
void scsi_start_queue(struct scsi_device *sdev)
{
struct request_queue *q = sdev->request_queue;
@@ -2857,6 +2826,56 @@
}
EXPORT_SYMBOL_GPL(scsi_target_unblock);
+int
+scsi_host_block(struct Scsi_Host *shost)
+{
+ struct scsi_device *sdev;
+ int ret = 0;
+
+ /*
+ * Call scsi_internal_device_block_nowait so we can avoid
+ * calling synchronize_rcu() for each LUN.
+ */
+ shost_for_each_device(sdev, shost) {
+ mutex_lock(&sdev->state_mutex);
+ ret = scsi_internal_device_block_nowait(sdev);
+ mutex_unlock(&sdev->state_mutex);
+ if (ret) {
+ scsi_device_put(sdev);
+ break;
+ }
+ }
+
+ /*
+ * SCSI never enables blk-mq's BLK_MQ_F_BLOCKING flag so
+ * calling synchronize_rcu() once is enough.
+ */
+ WARN_ON_ONCE(shost->tag_set.flags & BLK_MQ_F_BLOCKING);
+
+ if (!ret)
+ synchronize_rcu();
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(scsi_host_block);
+
+int
+scsi_host_unblock(struct Scsi_Host *shost, int new_state)
+{
+ struct scsi_device *sdev;
+ int ret = 0;
+
+ shost_for_each_device(sdev, shost) {
+ ret = scsi_internal_device_unblock(sdev, new_state);
+ if (ret) {
+ scsi_device_put(sdev);
+ break;
+ }
+ }
+ return ret;
+}
+EXPORT_SYMBOL_GPL(scsi_host_unblock);
+
/**
* scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
* @sgl: scatter-gather list