nvme: add Apple T2 ANS2 NVMe quirks

The Apple T2 (ANS2, PCI 106b:2005) requires several quirks:

- 128-byte submission queue entries (CC.IOSQES = 7)
- Single MSI vector, one IO queue
- Admin and IO queues share a CID table; IO CIDs offset by
  adminq.num_trackers to avoid overlap
- No async event support
- IDENTIFY CNS >= 2 rejected to avoid firmware confusion

Tested-on:
- MacBookPro16,2 (A2251)
- Mac mini 8,1 (A1993)
- Multiple Non-Apple computers

Reviewed by:	imp
Differential Revision:	https://reviews.freebsd.org/D57087

sys/dev/nvme/nvme_ctrlr.c

@@ -173,6 +173,10 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
 	num_entries = min(num_entries, mqes + 1);
 	num_entries = min(num_entries, max_entries);
 
+	/* SHARED_CID_SPACE: IO CIDs must fit within the shared CID table. */
+	if (ctrlr->quirks & QUIRK_APPLE_SHARED_CID_SPACE)
+		num_entries = min(num_entries, NVME_ADMIN_ENTRIES);
+
 	num_trackers = NVME_IO_TRACKERS;
 	TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
 
@@ -185,6 +189,10 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
 	 */
 	num_trackers = min(num_trackers, (num_entries-1));
 
+	if (ctrlr->quirks & QUIRK_APPLE_SHARED_CID_SPACE)
+		num_trackers = min(num_trackers,
+		    NVME_ADMIN_ENTRIES - ctrlr->adminq.num_trackers);
+
 	/*
 	 * Our best estimate for the maximum number of I/Os that we should
 	 * normally have in flight at one time. This should be viewed as a hint,
@@ -749,6 +757,11 @@ nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
 	struct nvme_async_event_request		*aer;
 	uint32_t				i;
 
+	if (ctrlr->quirks & QUIRK_APPLE_NO_ASYNC_EVENT) {
+		ctrlr->num_aers = 0;
+		return;
+	}
+
 	ctrlr->async_event_config = NVME_CRIT_WARN_ST_AVAILABLE_SPARE |
 	    NVME_CRIT_WARN_ST_DEVICE_RELIABILITY |
 	    NVME_CRIT_WARN_ST_READ_ONLY |

sys/dev/nvme/nvme_pci.c

@@ -93,6 +93,9 @@ static struct _pcsid
 	{ 0xa822144d,		0, 0, "Samsung PM1725a", QUIRK_DELAY_B4_CHK_RDY },
 	{ 0x07f015ad,		0, 0, "VMware NVMe Controller" },
 	{ 0x2003106b,		0, 0, "Apple S3X NVMe Controller" },
+	{ 0x2005106b,		0, 0, "Apple ANS2 NVMe Controller (T2)",
+	    QUIRK_APPLE_IDENTIFY_CNS_BROKEN | QUIRK_APPLE_SHARED_CID_SPACE |
+	    QUIRK_APPLE_NO_ASYNC_EVENT | QUIRK_APPLE_SINGLE_VECTOR },
 	{ 0x00000000,		0, 0, NULL  }
 };
 
@@ -131,6 +134,9 @@ nvme_pci_probe (device_t device)
 	if (ep->devid)
 		ctrlr->quirks = ep->quirks;
 
+	if (ctrlr->quirks & QUIRK_APPLE_IDENTIFY_CNS_BROKEN)
+		ctrlr->max_identify_cns = NVME_APPLE_ANS2_MAX_CNS;
+
 	if (ep->desc) {
 		device_set_desc(device, ep->desc);
 		return (BUS_PROBE_DEFAULT);
@@ -323,6 +329,15 @@ nvme_ctrlr_setup_interrupts(struct nvme_controller *ctrlr)
 	if (force_intx)
 		return (nvme_ctrlr_setup_shared(ctrlr, 0));
 
+	if (ctrlr->quirks & QUIRK_APPLE_SINGLE_VECTOR) {
+		int n = 1;
+		if (pci_alloc_msi(dev, &n) == 0) {
+			ctrlr->msi_count = n;
+			return (nvme_ctrlr_setup_shared(ctrlr, 1));
+		}
+		return (nvme_ctrlr_setup_shared(ctrlr, 0));
+	}
+
 	if (pci_msix_count(dev) == 0)
 		goto msi;
 

sys/dev/nvme/nvme_private.h

@@ -165,6 +165,8 @@ struct nvme_qpair {
 
 	uint32_t		num_entries;
 	uint32_t		num_trackers;
+	uint32_t		sqe_shift;	/* SQE size shift: sqes_max - 6; 0 = 64 bytes, 1 = 128 bytes */
+	uint16_t		cid_base;	/* CID offset for SHARED_TAGS IO queues */
 	uint32_t		sq_tdbl_off;
 	uint32_t		cq_hdbl_off;
 
@@ -220,10 +222,16 @@ struct nvme_controller {
 	int			domain;
 	uint32_t		ready_timeout_in_ms;
 	uint32_t		quirks;
+	uint8_t			max_identify_cns;	/* max CNS value for IDENTIFY (0 = no limit) */
 #define	QUIRK_DELAY_B4_CHK_RDY	1		/* Can't touch MMIO on disable */
 #define	QUIRK_DISABLE_TIMEOUT	2		/* Disable broken completion timeout feature */
 #define	QUIRK_INTEL_ALIGNMENT	4		/* Pre NVMe 1.3 performance alignment */
 #define QUIRK_AHCI		8		/* Attached via AHCI redirect */
+#define	QUIRK_APPLE_IDENTIFY_CNS_BROKEN		0x10	/* Reject IDENTIFY with CNS >= max_identify_cns */
+#define	NVME_APPLE_ANS2_MAX_CNS			1	/* T2: highest CNS accepted (Identify Controller) */
+#define	QUIRK_APPLE_SHARED_CID_SPACE		0x20	/* Admin/IO share a single CID table */
+#define	QUIRK_APPLE_NO_ASYNC_EVENT		0x40	/* Skip NVMe async event requests */
+#define	QUIRK_APPLE_SINGLE_VECTOR		0x80	/* Single MSI vector, one IO queue */
 
 	int			resource_id;
 	struct resource		*resource;
@@ -322,6 +330,12 @@ struct nvme_controller {
 	counter_u64_t			alignment_splits;
 };
 
+/*
+ * Access the idx'th submission queue entry.
+ * sqe_shift is sqes_max - 6: 0 for standard 64-byte SQEs, 1 for 128-byte.
+ */
+#define	NVME_SQE(qpair, idx)	(&(qpair)->cmd[(idx) << (qpair)->sqe_shift])
+
 #define nvme_mmio_offsetof(reg)						       \
 	offsetof(struct nvme_registers, reg)
 

sys/dev/nvme/nvme_qpair.c

@@ -231,7 +231,7 @@ nvme_qpair_complete_tracker(struct nvme_tracker *tr,
 		nvme_qpair_print_completion(qpair, cpl);
 	}
 
-	qpair->act_tr[cpl->cid] = NULL;
+	qpair->act_tr[cpl->cid - qpair->cid_base] = NULL;
 
 	KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n"));
 
@@ -305,7 +305,7 @@ nvme_qpair_manual_complete_tracker(
 	memset(&cpl, 0, sizeof(cpl));
 
 	cpl.sqid = qpair->id;
-	cpl.cid = tr->cid;
+	cpl.cid = qpair->cid_base + tr->cid;
 	cpl.status = nvme_qpair_make_status(sct, sc, dnr);
 	nvme_qpair_complete_tracker(tr, &cpl, print_on_error);
 }
@@ -433,8 +433,9 @@ _nvme_qpair_process_completions(struct nvme_qpair *qpair)
 		    NVME_STATUS_GET_P(status) == NVME_STATUS_GET_P(cpl.status),
 		    ("Phase unexpectedly inconsistent"));
 
-		if (cpl.cid < qpair->num_trackers)
-			tr = qpair->act_tr[cpl.cid];
+		if (cpl.cid >= qpair->cid_base &&
+		    cpl.cid < qpair->cid_base + qpair->num_trackers)
+			tr = qpair->act_tr[cpl.cid - qpair->cid_base];
 		else
 			tr = NULL;
 
@@ -530,6 +531,18 @@ nvme_qpair_construct(struct nvme_qpair *qpair,
 	qpair->num_trackers = num_trackers;
 	qpair->ctrlr = ctrlr;
 
+	/* sqes[7:4]: max SQE size exponent; admin always 64 bytes per spec. */
+	if (qpair->id != 0) {
+		uint8_t sqes_max = (ctrlr->cdata.sqes >> 4) & 0xf;
+		qpair->sqe_shift = (sqes_max > 6) ? (sqes_max - 6) : 0;
+	} else {
+		qpair->sqe_shift = 0;
+	}
+	if ((ctrlr->quirks & QUIRK_APPLE_SHARED_CID_SPACE) && qpair->id != 0)
+		qpair->cid_base = ctrlr->adminq.num_trackers;
+	else
+		qpair->cid_base = 0;
+
 	mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF);
 	mtx_init(&qpair->recovery, "nvme qpair recovery", NULL, MTX_DEF);
 
@@ -553,7 +566,7 @@ nvme_qpair_construct(struct nvme_qpair *qpair,
 	 * Each component must be page aligned, and individual PRP lists
 	 * cannot cross a page boundary.
 	 */
-	cmdsz = qpair->num_entries * sizeof(struct nvme_command);
+	cmdsz = qpair->num_entries * sizeof(struct nvme_command) << qpair->sqe_shift;
 	cmdsz = roundup2(cmdsz, ctrlr->page_size);
 	cplsz = qpair->num_entries * sizeof(struct nvme_completion);
 	cplsz = roundup2(cplsz, ctrlr->page_size);
@@ -987,7 +1000,8 @@ nvme_qpair_timeout(void *arg)
 				 * queue which will reset the card if it
 				 * times out.
 				 */
-				nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id,
+				nvme_ctrlr_cmd_abort(ctrlr,
+				    qpair->cid_base + tr->cid, qpair->id,
 				    nvme_abort_complete, tr);
 			} else {
 				/*
@@ -1040,7 +1054,7 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
 	mtx_assert(&qpair->lock, MA_OWNED);
 
 	req = tr->req;
-	req->cmd.cid = tr->cid;
+	req->cmd.cid = qpair->cid_base + tr->cid;
 	qpair->act_tr[tr->cid] = tr;
 	ctrlr = qpair->ctrlr;
 
@@ -1061,7 +1075,7 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
 		tr->deadline = SBT_MAX;
 
 	/* Copy the command from the tracker to the submission queue. */
-	memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd));
+	memcpy(NVME_SQE(qpair, qpair->sq_tail), &req->cmd, sizeof(req->cmd));
 
 	if (++qpair->sq_tail == qpair->num_entries)
 		qpair->sq_tail = 0;
@@ -1235,7 +1249,7 @@ nvme_qpair_reset(struct nvme_qpair *qpair)
 	qpair->phase = 1;
 
 	memset(qpair->cmd, 0,
-	    qpair->num_entries * sizeof(struct nvme_command));
+	    qpair->num_entries * sizeof(struct nvme_command) << qpair->sqe_shift);
 	memset(qpair->cpl, 0,
 	    qpair->num_entries * sizeof(struct nvme_completion));
 }

sys/dev/nvme/nvme_sim.c

@@ -96,6 +96,15 @@ nvme_sim_nvmeio(struct cam_sim *sim, union ccb *ccb)
 	struct nvme_controller *ctrlr;
 
 	ctrlr = sim2ctrlr(sim);
+
+	if (ctrlr->max_identify_cns != 0 &&
+	    nvmeio->cmd.opc == NVME_OPC_IDENTIFY &&
+	    (le32toh(nvmeio->cmd.cdw10) & 0xff) > ctrlr->max_identify_cns) {
+		nvmeio->ccb_h.status = CAM_REQ_INVALID;
+		xpt_done(ccb);
+		return;
+	}
+
 	payload = nvmeio->data_ptr;
 	size = nvmeio->dxfer_len;
 	/* SG LIST ??? */

sys/dev/nvme/nvme_sysctl.c

@@ -69,7 +69,7 @@ nvme_dump_queue(struct nvme_qpair *qpair)
 
 	printf("Submission queue:\n");
 	for (i = 0; i < qpair->num_entries; i++) {
-		cmd = &qpair->cmd[i];
+		cmd = NVME_SQE(qpair, i);
 		printf("%05d: ", i);
 		nvme_qpair_print_command(qpair, cmd);
 	}