From dafd43c90109c1b712b982c6e7f9f6ff32cd3d48 Mon Sep 17 00:00:00 2001 From: Nick Chan Date: Fri, 5 Jun 2026 15:20:56 +0800 Subject: [PATCH 1/3] nvme-apple: Allow maximum possible admin queue tags when Linear SQ is absent Apple NVMe controllers require tags of pending commands to not be shared across admin and IO queues. However, on Apple A11 without linear SQ, it is not possible for the IO queue to skip over some tags and must go from 0 to the configured maximum before wrapping around. As a result, in order to prevent tag collision, dynamic tag reservation while a command is submitted becomes neccessary. In this context, there is no reason to limit the admin queue's tag space, as it is not helpful in preventing tag collision. Signed-off-by: Nick Chan --- drivers/nvme/host/apple.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 9b9c6084609fa24..88a39bde3c6ae93 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1307,7 +1307,10 @@ static int apple_nvme_alloc_tagsets(struct apple_nvme *anv) anv->admin_tagset.ops = &apple_nvme_mq_admin_ops; anv->admin_tagset.nr_hw_queues = 1; - anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH; + if (anv->hw->has_lsq_nvmmu) + anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH; + else + anv->tagset.queue_depth = anv->hw->max_queue_depth - 1; anv->admin_tagset.timeout = NVME_ADMIN_TIMEOUT; anv->admin_tagset.numa_node = NUMA_NO_NODE; anv->admin_tagset.cmd_size = sizeof(struct apple_nvme_iod); From da64d586a9ce567210b721df4d19c34bc7dfcd64 Mon Sep 17 00:00:00 2001 From: yhavry <34289148+yhavry@users.noreply.github.com> Date: Thu, 4 Jun 2026 22:26:14 -0400 Subject: [PATCH 2/3] nvme-apple: serialize the controller-global command tag space on t8015 On t8015 (A11), the admin queue uses command tag 0 and the IO tagset is given no reserved tags, so the admin and IO queues can have tag 0 outstanding at the same time. When that happens the controller reports a duplicate-tag error for tag 0. We have no documentation for ANS2's tag handling on this generation, but the failure strongly suggests it treats the command tag as a single controller-wide resource rather than a per-queue one: making the tag unique across both queues makes the error go away. Enforce that with a controller-wide bitmap of active tags and: - reserve the tag before hardware submission; - if it is already active, return BLK_STS_RESOURCE so blk-mq retries rather than busy-waiting; - release it only after the CQ head has been acknowledged to ANS; - clear stale tag state on controller reset. Signed-off-by: Yuriy Havrylyuk Signed-off-by: yhavry <34289148+yhavry@users.noreply.github.com> --- drivers/nvme/host/apple.c | 62 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 39a2404eab794a1..bd8de0a33828cb5 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -203,6 +203,8 @@ struct apple_nvme { int irq; spinlock_t lock; + + unsigned long t8015_active_tags; }; static_assert(sizeof(struct nvme_command) == 64); @@ -290,6 +292,38 @@ static void apple_nvmmu_inval(struct apple_nvme_queue *q, unsigned int tag) "NVMMU TCB invalidation failed\n"); } +static bool apple_nvme_t8015_reserve_tag(struct apple_nvme *anv, + struct nvme_command *cmd) +{ + u16 tag; + + if (anv->hw->has_lsq_nvmmu) + return true; + + tag = nvme_tag_from_cid(cmd->common.command_id); + + if (WARN_ON_ONCE(tag >= BITS_PER_LONG)) + return false; + + return !test_and_set_bit(tag, &anv->t8015_active_tags); +} + +static void apple_nvme_t8015_release_cid(struct apple_nvme *anv, + __u16 command_id) +{ + u16 tag; + + if (anv->hw->has_lsq_nvmmu) + return; + + tag = nvme_tag_from_cid(command_id); + + if (WARN_ON_ONCE(tag >= BITS_PER_LONG)) + return; + + clear_bit(tag, &anv->t8015_active_tags); +} + static void apple_nvme_submit_cmd_t8015(struct apple_nvme_queue *q, struct nvme_command *cmd) { @@ -310,7 +344,6 @@ static void apple_nvme_submit_cmd_t8015(struct apple_nvme_queue *q, spin_unlock_irq(&anv->lock); } - static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, struct nvme_command *cmd) { @@ -652,9 +685,13 @@ static inline void apple_nvme_update_cq_head(struct apple_nvme_queue *q) static bool apple_nvme_poll_cq(struct apple_nvme_queue *q, struct io_comp_batch *iob) { + struct apple_nvme *anv = queue_to_apple_nvme(q); bool found = false; while (apple_nvme_cqe_pending(q)) { + struct nvme_completion *cqe = &q->cqes[q->cq_head]; + __u16 command_id; + found = true; /* @@ -662,11 +699,20 @@ static bool apple_nvme_poll_cq(struct apple_nvme_queue *q, * the cqe requires a full read memory barrier */ dma_rmb(); + + command_id = READ_ONCE(cqe->command_id); + apple_nvme_handle_cqe(q, iob, q->cq_head); apple_nvme_update_cq_head(q); + + if (!anv->hw->has_lsq_nvmmu) { + writel(q->cq_head, q->cq_db); + readl(q->cq_db); + apple_nvme_t8015_release_cid(anv, command_id); + } } - if (found) + if (found && anv->hw->has_lsq_nvmmu) writel(q->cq_head, q->cq_db); return found; @@ -790,10 +836,15 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, if (ret) return ret; + if (!apple_nvme_t8015_reserve_tag(anv, cmnd)) { + ret = BLK_STS_RESOURCE; + goto out_free_cmd; + } + if (blk_rq_nr_phys_segments(req)) { ret = apple_nvme_map_data(anv, req, cmnd); if (ret) - goto out_free_cmd; + goto out_release_tag; } nvme_start_request(req); @@ -805,6 +856,8 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, return BLK_STS_OK; +out_release_tag: + apple_nvme_t8015_release_cid(anv, cmnd->common.command_id); out_free_cmd: nvme_cleanup_cmd(req); return ret; @@ -1165,6 +1218,9 @@ static void apple_nvme_reset_work(struct work_struct *work) if (ret) goto out; + if (!anv->hw->has_lsq_nvmmu) + WRITE_ONCE(anv->t8015_active_tags, 0); + dev_dbg(anv->dev, "Starting admin queue"); apple_nvme_init_queue(&anv->adminq); nvme_unquiesce_admin_queue(&anv->ctrl); From c68f72ecdd91b633586570c9e6cb2b719c6252be Mon Sep 17 00:00:00 2001 From: yhavry <34289148+yhavry@users.noreply.github.com> Date: Fri, 5 Jun 2026 23:35:53 -0400 Subject: [PATCH 3/3] nvme-apple: serialize t8015 command tags across queues Signed-off-by: yhavry <34289148+yhavry@users.noreply.github.com> --- drivers/nvme/host/apple.c | 61 ++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index bd8de0a33828cb5..e88f6a5c9dae054 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -204,6 +204,14 @@ struct apple_nvme { int irq; spinlock_t lock; + /* + * t8015's ANS seems to treat the command tag as global across + * both the admin and I/O queues rather than per-queue. It's not + * clear why; our best guess is the firmware keys some internal + * state off the tag alone, so reusing one that's still live on + * the other queue upsets it. Track the in-flight tags here and + * hold off submitting until the tag is free. + */ unsigned long t8015_active_tags; }; @@ -292,15 +300,10 @@ static void apple_nvmmu_inval(struct apple_nvme_queue *q, unsigned int tag) "NVMMU TCB invalidation failed\n"); } -static bool apple_nvme_t8015_reserve_tag(struct apple_nvme *anv, +static bool apple_nvme_reserve_tag_t8015(struct apple_nvme *anv, struct nvme_command *cmd) { - u16 tag; - - if (anv->hw->has_lsq_nvmmu) - return true; - - tag = nvme_tag_from_cid(cmd->common.command_id); + u16 tag = nvme_tag_from_cid(cmd->common.command_id); if (WARN_ON_ONCE(tag >= BITS_PER_LONG)) return false; @@ -308,15 +311,10 @@ static bool apple_nvme_t8015_reserve_tag(struct apple_nvme *anv, return !test_and_set_bit(tag, &anv->t8015_active_tags); } -static void apple_nvme_t8015_release_cid(struct apple_nvme *anv, +static void apple_nvme_release_cid_t8015(struct apple_nvme *anv, __u16 command_id) { - u16 tag; - - if (anv->hw->has_lsq_nvmmu) - return; - - tag = nvme_tag_from_cid(command_id); + u16 tag = nvme_tag_from_cid(command_id); if (WARN_ON_ONCE(tag >= BITS_PER_LONG)) return; @@ -686,6 +684,7 @@ static bool apple_nvme_poll_cq(struct apple_nvme_queue *q, struct io_comp_batch *iob) { struct apple_nvme *anv = queue_to_apple_nvme(q); + unsigned long completed_tags = 0; bool found = false; while (apple_nvme_cqe_pending(q)) { @@ -706,15 +705,23 @@ static bool apple_nvme_poll_cq(struct apple_nvme_queue *q, apple_nvme_update_cq_head(q); if (!anv->hw->has_lsq_nvmmu) { - writel(q->cq_head, q->cq_db); - readl(q->cq_db); - apple_nvme_t8015_release_cid(anv, command_id); + u16 tag = nvme_tag_from_cid(command_id); + + if (!WARN_ON_ONCE(tag >= BITS_PER_LONG)) + __set_bit(tag, &completed_tags); } } - if (found && anv->hw->has_lsq_nvmmu) + if (found) writel(q->cq_head, q->cq_db); + if (!anv->hw->has_lsq_nvmmu && completed_tags) { + unsigned int tag; + + for_each_set_bit(tag, &completed_tags, BITS_PER_LONG) + clear_bit(tag, &anv->t8015_active_tags); + } + return found; } @@ -836,15 +843,20 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, if (ret) return ret; - if (!apple_nvme_t8015_reserve_tag(anv, cmnd)) { + if (!anv->hw->has_lsq_nvmmu && + !apple_nvme_reserve_tag_t8015(anv, cmnd)) { ret = BLK_STS_RESOURCE; goto out_free_cmd; } if (blk_rq_nr_phys_segments(req)) { ret = apple_nvme_map_data(anv, req, cmnd); - if (ret) - goto out_release_tag; + if (ret) { + if (!anv->hw->has_lsq_nvmmu) + apple_nvme_release_cid_t8015(anv, + cmnd->common.command_id); + goto out_free_cmd; + } } nvme_start_request(req); @@ -856,8 +868,6 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, return BLK_STS_OK; -out_release_tag: - apple_nvme_t8015_release_cid(anv, cmnd->common.command_id); out_free_cmd: nvme_cleanup_cmd(req); return ret; @@ -1359,10 +1369,7 @@ static int apple_nvme_alloc_tagsets(struct apple_nvme *anv) anv->admin_tagset.ops = &apple_nvme_mq_admin_ops; anv->admin_tagset.nr_hw_queues = 1; - if (anv->hw->has_lsq_nvmmu) - anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH; - else - anv->tagset.queue_depth = anv->hw->max_queue_depth - 1; + anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH; anv->admin_tagset.timeout = NVME_ADMIN_TIMEOUT; anv->admin_tagset.numa_node = NUMA_NO_NODE; anv->admin_tagset.cmd_size = sizeof(struct apple_nvme_iod);