From 9f90536c74b8172fc67cd977e5451f37a12462d5 Mon Sep 17 00:00:00 2001 From: Abdelkader Boudih Date: Sun, 14 Jun 2026 13:54:28 -0700 Subject: [PATCH] apple_bce/vhci: add T2 virtual USB host controller Implements a VHCI driver on top of the BCE transport: - Virtual USB bus registration via usb_controller - Port discovery and device enumeration - Control, interrupt, and bulk endpoint support - Firmware event handling with taskqueue - Suspend/resume via BCE mailbox Provides keyboard, trackpad, and Touch Bar access on T2 Macs. Tested-on: MacBookPro16,2 (A2251), Mac mini 8,1 (A1993) Reviewed by: adrian Differential Revision: https://reviews.freebsd.org/D57089 --- sys/conf/files.amd64 | 1 + sys/dev/apple_bce/apple_bce.c | 87 + sys/dev/apple_bce/apple_bce.h | 3 +- sys/dev/apple_bce/apple_bce_mailbox.c | 5 + sys/dev/apple_bce/apple_bce_vhci.c | 4821 +++++++++++++++++++++++ sys/dev/apple_bce/apple_bce_vhci.h | 251 ++ sys/dev/usb/controller/usb_controller.c | 1 + sys/modules/apple_bce/Makefile | 2 + 8 files changed, 5170 insertions(+), 1 deletion(-) create mode 100644 sys/dev/apple_bce/apple_bce_vhci.c create mode 100644 sys/dev/apple_bce/apple_bce_vhci.h diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index 4d37266d197..0b4a22eebc7 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -115,6 +115,7 @@ dev/amdgpio/amdgpio.c optional amdgpio dev/apple_bce/apple_bce.c optional apple_bce pci dev/apple_bce/apple_bce_mailbox.c optional apple_bce pci dev/apple_bce/apple_bce_queue.c optional apple_bce pci +dev/apple_bce/apple_bce_vhci.c optional apple_bce pci dev/asmc/asmc.c optional asmc isa dev/asmc/asmcmmio.c optional asmc isa dev/axgbe/if_axgbe_pci.c optional axp diff --git a/sys/dev/apple_bce/apple_bce.c b/sys/dev/apple_bce/apple_bce.c index 8700eddbc0f..2b0e6928ffc 100644 --- a/sys/dev/apple_bce/apple_bce.c +++ b/sys/dev/apple_bce/apple_bce.c @@ -32,10 +32,13 @@ #include "apple_bce.h" #include "apple_bce_mailbox.h" #include "apple_bce_queue.h" +#include "apple_bce_vhci.h" static int apple_bce_probe(device_t dev); static int apple_bce_attach(device_t dev); static int apple_bce_detach(device_t dev); +static int apple_bce_suspend(device_t dev); +static int apple_bce_resume(device_t dev); static void apple_bce_timestamp_cb(void *arg); static void apple_bce_timestamp_init(struct apple_bce_softc *sc); static void apple_bce_timestamp_start(struct apple_bce_softc *sc, @@ -516,6 +519,12 @@ apple_bce_attach(device_t dev) goto fail; device_printf(dev, "Apple T2 BCE initialized\n"); + + /* Create VHCI child for virtual USB */ + error = bce_vhci_attach(sc); + if (error != 0) + goto fail; + return (0); fail: @@ -531,6 +540,9 @@ apple_bce_detach(device_t dev) { struct apple_bce_softc *sc = device_get_softc(dev); + /* 0. Detach VHCI child first (before destroying parent resources) */ + bce_vhci_detach(sc); + /* 1. Stop timestamp */ if (sc->sc_bar4 != NULL && mtx_initialized(&sc->sc_timestamp_lock)) apple_bce_timestamp_stop(sc); @@ -616,10 +628,85 @@ apple_bce_detach(device_t dev) return (0); } +static int +apple_bce_suspend(device_t dev) +{ + struct apple_bce_softc *sc = device_get_softc(dev); + int error, restore_error; + + apple_bce_timestamp_stop(sc); + + error = bce_vhci_detach(sc); + if (error != 0) { + device_printf(dev, "failed to detach VHCI for suspend: %d\n", + error); + apple_bce_timestamp_start(sc, 0); + return (error); + } + + error = bce_mailbox_send(&sc->sc_mbox, + BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0), NULL, + BCE_MBOX_TIMEOUT_MS); + if (error != 0) { + device_printf(dev, + "failed to send SLEEP_NO_STATE mailbox command: %d\n", + error); + restore_error = bce_vhci_attach(sc); + if (restore_error != 0) { + device_printf(dev, + "failed to reattach VHCI after suspend error: %d\n", + restore_error); + } + apple_bce_timestamp_start(sc, 0); + return (error); + } + + return (0); +} + +static int +apple_bce_resume(device_t dev) +{ + struct apple_bce_softc *sc = device_get_softc(dev); + uint64_t reply; + int error; + + error = bce_mailbox_send(&sc->sc_mbox, + BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &reply, + BCE_MBOX_TIMEOUT_MS); + if (error != 0) { + device_printf(dev, + "failed to send RESTORE_NO_STATE mailbox command: %d\n", + error); + return (error); + } + + if (BCE_MB_TYPE(reply) != BCE_MB_RESTORE_NO_STATE) { + device_printf(dev, + "unexpected RESTORE_NO_STATE reply: type=%u val=0x%llx\n", + BCE_MB_TYPE(reply), + (unsigned long long)BCE_MB_VALUE(reply)); + return (EINVAL); + } + + error = bce_vhci_attach(sc); + if (error != 0) { + device_printf(dev, "failed to reattach VHCI after resume: %d\n", + error); + apple_bce_timestamp_start(sc, 0); + return (error); + } + + apple_bce_timestamp_start(sc, 0); + return (0); +} + static device_method_t apple_bce_methods[] = { DEVMETHOD(device_probe, apple_bce_probe), DEVMETHOD(device_attach, apple_bce_attach), DEVMETHOD(device_detach, apple_bce_detach), + DEVMETHOD(device_suspend, apple_bce_suspend), + DEVMETHOD(device_resume, apple_bce_resume), DEVMETHOD_END }; diff --git a/sys/dev/apple_bce/apple_bce.h b/sys/dev/apple_bce/apple_bce.h index 7c2198c7232..0e1999e48d5 100644 --- a/sys/dev/apple_bce/apple_bce.h +++ b/sys/dev/apple_bce/apple_bce.h @@ -27,7 +27,7 @@ #define BCE_PCI_DEVICE_T2 0x1801 #define BCE_MAX_QUEUE_COUNT 0x100 -#define BCE_MAX_CQ_COUNT 16 /* Max completion queues tracked */ +#define BCE_MAX_CQ_COUNT 64 /* Max completion queues tracked */ #define BCE_QUEUE_USER_MIN 2 #define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1) #define BCE_CMD_SIZE 0x40 @@ -286,6 +286,7 @@ struct apple_bce_softc { struct mtx sc_queues_lock; struct bce_queue_cq *sc_cq_list[BCE_MAX_CQ_COUNT]; struct bce_queue_sq *sc_int_sq_list[BCE_MAX_QUEUE_COUNT]; + device_t sc_vhci_dev; }; /* Inline helpers */ diff --git a/sys/dev/apple_bce/apple_bce_mailbox.c b/sys/dev/apple_bce/apple_bce_mailbox.c index abdb809b5c5..c19a01b7269 100644 --- a/sys/dev/apple_bce/apple_bce_mailbox.c +++ b/sys/dev/apple_bce/apple_bce_mailbox.c @@ -57,6 +57,11 @@ bce_mailbox_send(struct bce_mailbox *mb, uint64_t msg, uint64_t *recv, bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 8, 0); bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 12, 0); + if (recv == NULL) { + atomic_store_int(&mb->status, 0); + return (0); + } + /* Wait for interrupt-driven reply */ if (sema_timedwait(&mb->mb_cmpl, hz * timeout_ms / 1000) != 0) { /* Timeout -- reset to idle */ diff --git a/sys/dev/apple_bce/apple_bce_vhci.c b/sys/dev/apple_bce/apple_bce_vhci.c new file mode 100644 index 00000000000..b8dbc9638b3 --- /dev/null +++ b/sys/dev/apple_bce/apple_bce_vhci.c @@ -0,0 +1,4821 @@ +/*- + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Apple T2 BCE Virtual USB Host Controller Interface (VHCI). + */ + +#ifdef USB_GLOBAL_INCLUDE_FILE +#include USB_GLOBAL_INCLUDE_FILE +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apple_bce.h" +#include "apple_bce_queue.h" +#include "apple_bce_vhci.h" + +/* + * VHCI softc, defined here because it depends on USB headers. + */ +struct bce_vhci_softc { + struct usb_bus sc_bus; /* Must be first */ + struct usb_device *sc_devices[BCE_VHCI_MAX_DEVICES]; + struct apple_bce_softc *sc_bce; + device_t sc_dev; + + /* Controller state */ + uint32_t sc_port_mask; + uint8_t sc_port_count; + int sc_started; + + /* Port state */ + uint32_t sc_port_status[BCE_VHCI_MAX_PORTS]; + uint32_t sc_port_change[BCE_VHCI_MAX_PORTS]; + uint8_t sc_port_power[BCE_VHCI_MAX_PORTS]; + + /* Hub scratch buffer (for descriptor/status responses) */ + uint8_t sc_hub_idata[32]; + + /* Message queues (host -> device) */ + struct bce_vhci_msg_queue msg_commands; + struct bce_vhci_msg_queue msg_system; + struct bce_vhci_msg_queue msg_isochronous; + struct bce_vhci_msg_queue msg_interrupt; + struct bce_vhci_msg_queue msg_asynchronous; + + /* Event queues (device -> host), share a single CQ */ + struct bce_queue_cq *ev_cq; + struct bce_vhci_evt_queue ev_commands; + struct bce_vhci_evt_queue ev_system; + struct bce_vhci_evt_queue ev_isochronous; + struct bce_vhci_evt_queue ev_interrupt; + struct bce_vhci_evt_queue ev_asynchronous; + + /* Command execution (synchronous, wraps msg_commands) */ + struct bce_vhci_cmd_queue cmd; + + /* Queue ID bitmap (256 bits = BCE_MAX_QUEUE_COUNT) */ + uint32_t sc_qid_bitmap[8]; + + /* Per-device state (indexed by firmware device ID) */ + struct bce_vhci_device sc_devs[BCE_VHCI_MAX_DEVICES]; + uint8_t sc_port_to_dev[BCE_VHCI_MAX_PORTS]; + + /* Deferred firmware event processing (from ev_commands) */ + struct task sc_fwevt_task; + volatile int sc_detaching; /* Teardown guard */ + + /* + * Firmware event mailbox: ISR copies events here, task processes. + * Protected by sc_fwevt_lock. Ring of BCE_VHCI_EVT_PENDING entries. + */ + struct mtx sc_fwevt_lock; +#define BCE_VHCI_FWEVT_RING (BCE_VHCI_EVT_PENDING + 1) + struct { + struct bce_vhci_message msg; + int needs_reply; + } sc_fwevt_ring[BCE_VHCI_FWEVT_RING]; + uint32_t sc_fwevt_prod; + uint32_t sc_fwevt_cons; + + /* Spinlock for msg_asynchronous writes (ISR + taskqueue context) */ + struct mtx sc_async_lock; + + /* Deferred endpoint reset (cannot sleep in pipe_start) */ + struct task sc_reset_task; + + /* Deferred endpoint create (cannot sleep in pipe_start) */ + struct task sc_create_task; + + /* Deferred port status change (ISR cannot call cmd_execute) */ + struct task sc_port_chg_task; + volatile uint32_t sc_port_chg_mask; +}; + +/* Command timeout (ticks) */ +#define BCE_VHCI_CMD_TIMEOUT_SHORT (hz * 2) +#define BCE_VHCI_CMD_TIMEOUT_LONG (hz * 30) + +static usb_handle_req_t bce_vhci_roothub_exec; +static void bce_vhci_endpoint_init(struct usb_device *udev, + struct usb_endpoint_descriptor *edesc, struct usb_endpoint *ep); +static void bce_vhci_xfer_setup(struct usb_setup_params *parm); +static void bce_vhci_xfer_unsetup(struct usb_xfer *xfer); +static void bce_vhci_get_dma_delay(struct usb_device *udev, uint32_t *pus); + +static void bce_vhci_pipe_open(struct usb_xfer *xfer); +static void bce_vhci_pipe_close(struct usb_xfer *xfer); +static void bce_vhci_pipe_enter(struct usb_xfer *xfer); +static void bce_vhci_pipe_start(struct usb_xfer *xfer); + +static int bce_vhci_probe(device_t dev); +static int bce_vhci_attach_dev(device_t dev); +static int bce_vhci_detach_dev(device_t dev); + +static int bce_vhci_alloc_qid(struct bce_vhci_softc *vhci); +static void bce_vhci_free_qid(struct bce_vhci_softc *vhci, int qid); +static int bce_vhci_create_queues(struct bce_vhci_softc *vhci); +static void bce_vhci_destroy_queues(struct bce_vhci_softc *vhci); +static int bce_vhci_start_controller(struct bce_vhci_softc *vhci); +static void bce_vhci_msg_queue_completion(struct bce_queue_sq *sq); +static void bce_vhci_ev_cmd_completion(struct bce_queue_sq *sq); +static void bce_vhci_ev_system_completion(struct bce_queue_sq *sq); +static void bce_vhci_ev_generic_completion(struct bce_queue_sq *sq); +static void bce_vhci_cmd_deliver_completion(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static void bce_vhci_handle_port_status_change(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static void bce_vhci_evt_queue_submit_pending(struct bce_vhci_softc *vhci, + struct bce_vhci_evt_queue *eq, uint32_t count); + +static int bce_vhci_device_create(struct bce_vhci_softc *vhci, uint8_t port); +static void bce_vhci_device_destroy(struct bce_vhci_softc *vhci, uint8_t port); +static int bce_vhci_endpoint_create(struct bce_vhci_softc *vhci, + struct bce_vhci_device *dev, uint8_t ep_addr, + struct usb_endpoint_descriptor *edesc); +static void bce_vhci_endpoint_destroy(struct bce_vhci_softc *vhci, + struct bce_vhci_device *dev, uint8_t ep_addr); +static void bce_vhci_handle_transfer_request(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static void bce_vhci_complete_ctrl_locked(struct bce_vhci_softc *vhci, + struct bce_vhci_transfer_queue *tq, struct bce_vhci_message *msg); +static void bce_vhci_handle_ctrl_status(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static uint16_t bce_vhci_handle_endpoint_req_state(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static uint16_t bce_vhci_handle_endpoint_set_state(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg); +static void bce_vhci_fwevt_task(void *arg, int pending); +static void bce_vhci_send_fw_event_reply(struct bce_vhci_softc *vhci, + struct bce_vhci_message *req, uint16_t status); +static void bce_vhci_tq_completion(struct bce_queue_sq *sq); +static void bce_vhci_reset_task(void *arg, int pending); +static void bce_vhci_create_task(void *arg, int pending); +static void bce_vhci_port_chg_task(void *arg, int pending); +static int bce_vhci_cmd_execute(struct bce_vhci_softc *vhci, + struct bce_vhci_message *req, struct bce_vhci_message *reply, + int timeout_ticks); + +/* + * Convert USB endpoint address to tq[] index. + * ep0 (0x00) maps to index 0. For other endpoints, IN and OUT get + * separate slots: OUT 0x01 -> 1, IN 0x81 -> 2, OUT 0x02 -> 3, etc. + * Maximum index is 30 (ep 0x8F), fits in BCE_VHCI_MAX_ENDPOINTS (32). + */ +static inline uint8_t +bce_vhci_ep_index(uint8_t ep_addr) +{ + uint8_t num; + + num = ep_addr & 0x0F; + if (num == 0) + return (0); + return (num * 2 - ((ep_addr & 0x80) ? 0 : 1)); +} + +static const struct usb_bus_methods bce_vhci_bus_methods = { + .roothub_exec = bce_vhci_roothub_exec, + .endpoint_init = bce_vhci_endpoint_init, + .xfer_setup = bce_vhci_xfer_setup, + .xfer_unsetup = bce_vhci_xfer_unsetup, + .get_dma_delay = bce_vhci_get_dma_delay, +}; + +/* + * Generic pipe methods (all transfer types for now). + */ +static const struct usb_pipe_methods bce_vhci_pipe_methods = { + .open = bce_vhci_pipe_open, + .close = bce_vhci_pipe_close, + .enter = bce_vhci_pipe_enter, + .start = bce_vhci_pipe_start, +}; + +/* + * Device methods. + */ +static device_method_t bce_vhci_methods[] = { + DEVMETHOD(device_probe, bce_vhci_probe), + DEVMETHOD(device_attach, bce_vhci_attach_dev), + DEVMETHOD(device_detach, bce_vhci_detach_dev), + DEVMETHOD(device_suspend, bus_generic_suspend), + DEVMETHOD(device_resume, bus_generic_resume), + DEVMETHOD(device_shutdown, bus_generic_shutdown), + + /* Bus interface for usbus child */ + DEVMETHOD(bus_print_child, bus_generic_print_child), + DEVMETHOD_END +}; + +static driver_t bce_vhci_driver = { + .name = "bce_vhci", + .methods = bce_vhci_methods, + .size = sizeof(struct bce_vhci_softc), +}; + +DRIVER_MODULE(bce_vhci, apple_bce, bce_vhci_driver, 0, 0); +MODULE_DEPEND(bce_vhci, usb, 1, 1, 1); + +/* + * Hub descriptor (USB 2.0 hub with per-port power switching) + */ + +/* Hub descriptor built dynamically in roothub_exec (port count varies) */ + +/* Device descriptor for the root hub */ +static const struct usb_device_descriptor bce_vhci_devd = { + .bLength = sizeof(struct usb_device_descriptor), + .bDescriptorType = UDESC_DEVICE, + .bcdUSB = { 0x00, 0x02 }, /* USB 2.0 */ + .bDeviceClass = UDCLASS_HUB, + .bDeviceSubClass = UDSUBCLASS_HUB, + .bDeviceProtocol = UDPROTO_HSHUBSTT, + .bMaxPacketSize = 64, + .idVendor = { 0x6b, 0x10 }, /* Apple 0x106b */ + .idProduct = { 0x01, 0x18 }, /* T2 BCE 0x1801 */ + .bcdDevice = { 0x00, 0x01 }, /* 1.00 */ + .iManufacturer = 1, + .iProduct = 2, + .bNumConfigurations = 1, +}; + +static const struct usb_device_qualifier bce_vhci_odevd = { + .bLength = sizeof(struct usb_device_qualifier), + .bDescriptorType = UDESC_DEVICE_QUALIFIER, + .bcdUSB = { 0x00, 0x02 }, + .bDeviceClass = UDCLASS_HUB, + .bDeviceSubClass = UDSUBCLASS_HUB, + .bDeviceProtocol = UDPROTO_HSHUBSTT, + .bMaxPacketSize0 = 0, + .bNumConfigurations = 0, +}; + +/* Configuration descriptor + interface + endpoint */ +/* 9 + 9 + 7 = 25 bytes */ +static const uint8_t bce_vhci_confd[] = { + /* Configuration descriptor */ + 0x09, 0x02, /* bLength, bDescriptorType */ + 0x19, 0x00, /* wTotalLength = 25 */ + 0x01, 0x01, 0x00, 0xC0, 0x00, /* nIntf, cfgVal, iCfg */ + /* Interface descriptor */ + 0x09, 0x04, /* bLength, bDescriptorType */ + 0x00, 0x00, 0x01, 0x09, 0x00, 0x01, 0x00, + /* Endpoint descriptor (interrupt IN ep1) */ + 0x07, 0x05, /* bLength, bDescriptorType */ + 0x81, 0x03, 0x08, 0x00, 0xFF, /* addr, attr, maxPkt, interval */ +}; + +struct bce_vhci_dma_cb_arg { + bus_addr_t addr; + int error; +}; + +static void +bce_vhci_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + struct bce_vhci_dma_cb_arg *cb = arg; + + cb->error = error; + if (error == 0) + cb->addr = segs[0].ds_addr; +} + +/* + * Allocate a CQ + SQ pair and DMA message buffer for a host->device queue. + * Register it with firmware under the given name. + */ +static int +bce_vhci_msg_queue_create(struct bce_vhci_softc *vhci, + struct bce_vhci_msg_queue *mq, const char *name, int cq_qid, int sq_qid, + bce_sq_completion_fn compl_fn, void *compl_arg) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + struct bce_vhci_dma_cb_arg cb; + struct bce_queue_memcfg cfg; + uint32_t el_count = BCE_VHCI_MSG_QUEUE_EL; + uint32_t status; + int error, i; + + memset(mq, 0, sizeof(*mq)); + mq->el_count = el_count; + + /* Allocate CQ */ + mq->cq = bce_alloc_cq(sc, cq_qid, el_count); + if (mq->cq == NULL) + return (ENOMEM); + + /* Register CQ with firmware via command path */ + bce_get_cq_memcfg(mq->cq, &cfg); + /* CQ interrupt vector = 4 (DMA MSI) */ + cfg.vector_or_cq = 4; + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, NULL, 0); + if (status != 0) { + device_printf(vhci->sc_dev, + "failed to register CQ %d for %s: %u\n", + cq_qid, name, status); + error = EIO; + goto fail_cq; + } + + /* Register CQ in parent's dispatch tables */ + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[cq_qid] = mq->cq; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == NULL) { + sc->sc_cq_list[i] = mq->cq; + break; + } + } + if (i == BCE_MAX_CQ_COUNT) { + sc->sc_queues[cq_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + device_printf(vhci->sc_dev, + "CQ list full for %s\n", name); + error = ENOSPC; + goto fail_cq_reg; + } + mtx_unlock(&sc->sc_queues_lock); + + /* Allocate SQ (element size = bce_qe_submission = 32 bytes) */ + mq->sq = bce_alloc_sq(sc, sq_qid, + sizeof(struct bce_qe_submission), el_count, + compl_fn, compl_arg); + if (mq->sq == NULL) { + error = ENOMEM; + goto fail_cq_reg; + } + + /* Register SQ with firmware under the given name */ + bce_get_sq_memcfg(mq->sq, mq->cq, &cfg); + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 1); + if (status != 0) { + device_printf(vhci->sc_dev, + "failed to register SQ %d (%s): %u\n", + sq_qid, name, status); + error = EIO; + goto fail_sq; + } + + /* Register SQ in parent's dispatch tables */ + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[sq_qid] = mq->sq; + sc->sc_int_sq_list[sq_qid] = mq->sq; + mtx_unlock(&sc->sc_queues_lock); + + /* Allocate DMA-coherent message buffer */ + error = bus_dma_tag_create(sc->sc_dma_tag, + 4, 0, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, + NULL, NULL, + el_count * sizeof(struct bce_vhci_message), 1, + el_count * sizeof(struct bce_vhci_message), + BUS_DMA_WAITOK, + NULL, NULL, + &mq->dma_tag); + if (error != 0) + goto fail_sq_reg; + + error = bus_dmamem_alloc(mq->dma_tag, (void **)&mq->data, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, + &mq->dma_map); + if (error != 0) + goto fail_dma_tag; + + error = bus_dmamap_load(mq->dma_tag, mq->dma_map, mq->data, + el_count * sizeof(struct bce_vhci_message), + bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK); + if (error != 0 || cb.error != 0) { + error = error != 0 ? error : cb.error; + goto fail_dma_mem; + } + mq->dma_addr = cb.addr; + + return (0); + +fail_dma_mem: + bus_dmamem_free(mq->dma_tag, mq->data, mq->dma_map); +fail_dma_tag: + bus_dma_tag_destroy(mq->dma_tag); +fail_sq_reg: + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, sq_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[sq_qid] = NULL; + sc->sc_int_sq_list[sq_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); +fail_sq: + bce_free_sq(sc, mq->sq); + mq->sq = NULL; +fail_cq_reg: + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, cq_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[cq_qid] = NULL; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == mq->cq) { + sc->sc_cq_list[i] = NULL; + break; + } + } + mtx_unlock(&sc->sc_queues_lock); +fail_cq: + bce_free_cq(sc, mq->cq); + mq->cq = NULL; + return (error); +} + +static void +bce_vhci_msg_queue_destroy(struct bce_vhci_softc *vhci, + struct bce_vhci_msg_queue *mq) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + int i; + + if (mq->cq == NULL) + return; + + /* + * Unregister and free SQ before releasing the DMA buffer it + * references + */ + if (mq->sq != NULL) { + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, mq->sq->qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[mq->sq->qid] = NULL; + sc->sc_int_sq_list[mq->sq->qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + bce_free_sq(sc, mq->sq); + mq->sq = NULL; + } + + /* Free DMA message buffer */ + if (mq->data != NULL) { + bus_dmamap_unload(mq->dma_tag, mq->dma_map); + bus_dmamem_free(mq->dma_tag, mq->data, mq->dma_map); + bus_dma_tag_destroy(mq->dma_tag); + mq->data = NULL; + } + + /* Unregister and free CQ */ + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, mq->cq->qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[mq->cq->qid] = NULL; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == mq->cq) { + sc->sc_cq_list[i] = NULL; + break; + } + } + mtx_unlock(&sc->sc_queues_lock); + bce_free_cq(sc, mq->cq); + mq->cq = NULL; +} + +/* + * Write a message to a host->device queue. + * Caller must have reserved a submission slot. + */ +static void +bce_vhci_msg_queue_write(struct bce_vhci_softc *vhci, + struct bce_vhci_msg_queue *mq, struct bce_vhci_message *msg) +{ + struct bce_qe_submission *s; + uint32_t sidx; + + sidx = mq->sq->tail; + s = bce_next_submission(mq->sq); + + /* Copy message into DMA buffer slot and sync for device access */ + mq->data[sidx] = *msg; + bus_dmamap_sync(mq->dma_tag, mq->dma_map, BUS_DMASYNC_PREWRITE); + + /* Fill SQ entry pointing to the DMA buffer slot */ + s->length = sizeof(struct bce_vhci_message); + s->addr = mq->dma_addr + + sidx * sizeof(struct bce_vhci_message); + s->segl_addr = 0; + s->segl_length = 0; + + bce_submit_to_device(vhci->sc_bce, mq->sq); +} + +/* + * Message queue completion: consume completions and free slots. + */ +static void +bce_vhci_msg_queue_completion(struct bce_queue_sq *sq) +{ + struct bce_vhci_msg_queue *mq = sq->userdata; + + while (sq->completion_cidx != sq->completion_tail) { + sq->completion_cidx = + (sq->completion_cidx + 1) % sq->el_count; + bce_notify_submission_complete(sq); + } + bus_dmamap_sync(mq->dma_tag, mq->dma_map, BUS_DMASYNC_POSTWRITE); +} + +/* + * Allocate an SQ (paired with the shared ev_cq) and DMA buffer for a + * device->host event queue. Register with firmware and pre-submit + * receive buffers. + */ +static int +bce_vhci_evt_queue_create(struct bce_vhci_softc *vhci, + struct bce_vhci_evt_queue *eq, const char *name, int sq_qid, + bce_sq_completion_fn compl_fn) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + struct bce_vhci_dma_cb_arg cb; + struct bce_queue_memcfg cfg; + uint32_t el_count = BCE_VHCI_EVT_QUEUE_EL; + uint32_t status; + int error; + + memset(eq, 0, sizeof(*eq)); + eq->el_count = el_count; + eq->userdata = vhci; + + /* Allocate SQ (shared CQ = vhci->ev_cq) */ + eq->sq = bce_alloc_sq(sc, sq_qid, + sizeof(struct bce_qe_submission), el_count, + compl_fn, eq); + if (eq->sq == NULL) + return (ENOMEM); + + /* Register SQ with firmware (direction = from device = 0) */ + bce_get_sq_memcfg(eq->sq, vhci->ev_cq, &cfg); + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 0); + if (status != 0) { + device_printf(vhci->sc_dev, + "failed to register event SQ %d (%s): %u\n", + sq_qid, name, status); + error = EIO; + goto fail_sq; + } + + /* Register SQ in dispatch tables */ + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[sq_qid] = eq->sq; + sc->sc_int_sq_list[sq_qid] = eq->sq; + mtx_unlock(&sc->sc_queues_lock); + + /* Allocate DMA-coherent receive buffer */ + error = bus_dma_tag_create(sc->sc_dma_tag, + 4, 0, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, + NULL, NULL, + el_count * sizeof(struct bce_vhci_message), 1, + el_count * sizeof(struct bce_vhci_message), + BUS_DMA_WAITOK, + NULL, NULL, + &eq->dma_tag); + if (error != 0) + goto fail_sq_reg; + + error = bus_dmamem_alloc(eq->dma_tag, (void **)&eq->data, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, + &eq->dma_map); + if (error != 0) + goto fail_dma_tag; + + error = bus_dmamap_load(eq->dma_tag, eq->dma_map, eq->data, + el_count * sizeof(struct bce_vhci_message), + bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK); + if (error != 0 || cb.error != 0) { + error = error != 0 ? error : cb.error; + goto fail_dma_mem; + } + eq->dma_addr = cb.addr; + + /* Pre-submit receive buffers */ + bce_vhci_evt_queue_submit_pending(vhci, eq, BCE_VHCI_EVT_PENDING); + + return (0); + +fail_dma_mem: + bus_dmamem_free(eq->dma_tag, eq->data, eq->dma_map); +fail_dma_tag: + bus_dma_tag_destroy(eq->dma_tag); +fail_sq_reg: + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, sq_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[sq_qid] = NULL; + sc->sc_int_sq_list[sq_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); +fail_sq: + bce_free_sq(sc, eq->sq); + eq->sq = NULL; + return (error); +} + +static void +bce_vhci_evt_queue_destroy(struct bce_vhci_softc *vhci, + struct bce_vhci_evt_queue *eq) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + + if (eq->sq == NULL) + return; + + /* Unregister SQ from dispatch tables FIRST to stop IRQ callbacks */ + bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, eq->sq->qid); + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, eq->sq->qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[eq->sq->qid] = NULL; + sc->sc_int_sq_list[eq->sq->qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + + /* Now safe to free DMA buffer; no IRQ can reference it */ + if (eq->data != NULL) { + bus_dmamap_unload(eq->dma_tag, eq->dma_map); + bus_dmamem_free(eq->dma_tag, eq->data, eq->dma_map); + bus_dma_tag_destroy(eq->dma_tag); + eq->data = NULL; + } + bce_free_sq(sc, eq->sq); + eq->sq = NULL; +} + +/* + * Submit empty receive buffers to an event queue so firmware can + * write messages into them. + */ +static void +bce_vhci_evt_queue_submit_pending(struct bce_vhci_softc *vhci, + struct bce_vhci_evt_queue *eq, uint32_t count) +{ + struct bce_qe_submission *s; + uint32_t idx; + + bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_PREREAD); + + while (count-- > 0) { + if (bce_reserve_submission(eq->sq) != 0) { + device_printf(vhci->sc_dev, + "cannot reserve event submission\n"); + break; + } + idx = eq->sq->tail; + s = bce_next_submission(eq->sq); + s->length = sizeof(struct bce_vhci_message); + s->addr = eq->dma_addr + + idx * sizeof(struct bce_vhci_message); + s->segl_addr = 0; + s->segl_length = 0; + } + bce_submit_to_device(vhci->sc_bce, eq->sq); +} + +/* + * Enqueue a firmware event into sc_fwevt_ring for deferred processing. + * Called from ISR context; returns 0 on success, -1 if ring is full. + */ +static int +bce_vhci_fwevt_enqueue(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg, int needs_reply) +{ + uint32_t next_prod; + + mtx_lock_spin(&vhci->sc_fwevt_lock); + next_prod = (vhci->sc_fwevt_prod + 1) % BCE_VHCI_FWEVT_RING; + if (next_prod == vhci->sc_fwevt_cons) { + mtx_unlock_spin(&vhci->sc_fwevt_lock); + device_printf(vhci->sc_dev, + "fwevt ring full, dropping 0x%04x\n", msg->cmd); + return (-1); + } + vhci->sc_fwevt_ring[vhci->sc_fwevt_prod].msg = *msg; + vhci->sc_fwevt_ring[vhci->sc_fwevt_prod].needs_reply = needs_reply; + vhci->sc_fwevt_prod = next_prod; + mtx_unlock_spin(&vhci->sc_fwevt_lock); + + if (vhci->sc_detaching == 0) + taskqueue_enqueue(taskqueue_thread, &vhci->sc_fwevt_task); + return (0); +} + +/* + * Generic event queue completion: read messages and resubmit buffers. + * Used for system, isochronous, interrupt, and asynchronous event queues. + */ +static void +bce_vhci_ev_generic_completion(struct bce_queue_sq *sq) +{ + struct bce_vhci_evt_queue *eq = sq->userdata; + struct bce_vhci_softc *vhci = eq->userdata; + struct bce_vhci_message *msg; + uint32_t cnt = 0; + + bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_POSTREAD); + + while (sq->completion_cidx != sq->completion_tail) { + struct bce_sq_completion_data *cd; + + cd = &sq->completion_data[sq->completion_cidx]; + if (cd->status == BCE_COMP_ABORTED) { + sq->completion_cidx = + (sq->completion_cidx + 1) % sq->el_count; + bce_notify_submission_complete(sq); + cnt++; + continue; + } + + msg = &eq->data[sq->head]; + /* + * Route events to appropriate handlers. + * Strip 0x4000 flag; firmware uses it as a + * variant marker. + */ + if (msg->cmd & BCE_VHCI_CMD_REPLY_FLAG) + bce_vhci_cmd_deliver_completion(vhci, msg); + else { + uint16_t base_cmd = msg->cmd & + ~BCE_VHCI_CMD_CANCEL_FLAG; + + if (base_cmd == BCE_VHCI_CMD_PORT_STATUS_CHANGE) + bce_vhci_handle_port_status_change(vhci, + msg); + else if (base_cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) + bce_vhci_handle_transfer_request(vhci, msg); + else if (base_cmd == + BCE_VHCI_CMD_CTRL_TRANSFER_STATUS) + bce_vhci_handle_ctrl_status(vhci, msg); + else if (base_cmd == + BCE_VHCI_CMD_ENDPOINT_REQ_STATE || + base_cmd == + BCE_VHCI_CMD_ENDPOINT_SET_STATE) + bce_vhci_fwevt_enqueue(vhci, msg, 0); + } + + sq->completion_cidx = + (sq->completion_cidx + 1) % sq->el_count; + bce_notify_submission_complete(sq); + cnt++; + } + + if (cnt > 0) + bce_vhci_evt_queue_submit_pending(vhci, eq, cnt); +} + +/* + * Event queue completion for the firmware command channel (ev_commands). + * + * This ISR callback is the sole consumer of the ev_commands SQ ring. + * Command replies are delivered inline (semaphore post, ISR-safe). + * Firmware events are copied into sc_fwevt_ring and deferred to + * sc_fwevt_task which handles them in taskqueue_thread context + * (needed because ENDP_PAUSED handling calls bce_cmd_flush_queue). + */ +static void +bce_vhci_ev_cmd_completion(struct bce_queue_sq *sq) +{ + struct bce_vhci_evt_queue *eq = sq->userdata; + struct bce_vhci_softc *vhci = eq->userdata; + struct bce_vhci_message *msg; + uint32_t cnt = 0; + + bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_POSTREAD); + + while (sq->completion_cidx != sq->completion_tail) { + struct bce_sq_completion_data *cd; + + cd = &sq->completion_data[sq->completion_cidx]; + if (cd->status == BCE_COMP_ABORTED) { + sq->completion_cidx = + (sq->completion_cidx + 1) % sq->el_count; + bce_notify_submission_complete(sq); + cnt++; + continue; + } + + msg = &eq->data[sq->head]; + + if (msg->cmd & BCE_VHCI_CMD_REPLY_FLAG) { + /* Command reply: deliver inline (semaphore post) */ + bce_vhci_cmd_deliver_completion(vhci, msg); + } else { + /* Firmware event: defer to taskqueue */ + bce_vhci_fwevt_enqueue(vhci, msg, 1); + } + + sq->completion_cidx = + (sq->completion_cidx + 1) % sq->el_count; + bce_notify_submission_complete(sq); + cnt++; + } + + if (cnt > 0) { + bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_PREREAD); + bce_vhci_evt_queue_submit_pending(vhci, eq, cnt); + } + +} + +/* + * System event queue completion: handles command replies and + * port status change notifications. + */ +static void +bce_vhci_ev_system_completion(struct bce_queue_sq *sq) +{ + + /* Route through generic handler which checks for both */ + bce_vhci_ev_generic_completion(sq); +} + +/* + * Taskqueue handler for firmware events on ev_commands. + * + * Processes ENDPOINT_REQ_STATE / ENDPOINT_SET_STATE events from + * process context (not ISR). Implements cancel-pair detection: + * if two consecutive events are cmd + cmd|0x4000 with same param1, + * both are consumed with a single ABORT reply. + * + * Normal events are handled and replied with SUCCESS on msg_system. + */ +static void +bce_vhci_fwevt_task(void *arg, int pending __unused) +{ + struct bce_vhci_softc *vhci = arg; + struct bce_vhci_message msg; + + if (vhci->sc_detaching) + return; + + /* + * Process firmware events from the mailbox ring. + * The ISR is the sole consumer of the ev_commands SQ ring and + * copies events here; we process them in taskqueue context. + */ + for (;;) { + uint16_t result; + int needs_reply; + + mtx_lock_spin(&vhci->sc_fwevt_lock); + if (vhci->sc_fwevt_cons == vhci->sc_fwevt_prod) { + mtx_unlock_spin(&vhci->sc_fwevt_lock); + break; + } + msg = vhci->sc_fwevt_ring[vhci->sc_fwevt_cons].msg; + needs_reply = + vhci->sc_fwevt_ring[vhci->sc_fwevt_cons].needs_reply; + vhci->sc_fwevt_cons = (vhci->sc_fwevt_cons + 1) % + BCE_VHCI_FWEVT_RING; + mtx_unlock_spin(&vhci->sc_fwevt_lock); + + if (msg.cmd & BCE_VHCI_CMD_CANCEL_FLAG) { + /* Firmware cancel; reply ABORT */ + result = BCE_VHCI_ABORT; + } else if (msg.cmd == BCE_VHCI_CMD_ENDPOINT_REQ_STATE) + result = bce_vhci_handle_endpoint_req_state(vhci, &msg); + else if (msg.cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) + result = bce_vhci_handle_endpoint_set_state(vhci, &msg); + else { + device_printf(vhci->sc_dev, + "unhandled fw event: 0x%04x\n", msg.cmd); + result = BCE_VHCI_BAD_ARGUMENT; + } + if (needs_reply) + bce_vhci_send_fw_event_reply(vhci, &msg, result); + } +} + +/* + * Deliver a firmware reply to the synchronous command waiter. + */ +static void +bce_vhci_cmd_deliver_completion(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg) +{ + struct bce_vhci_cmd_queue *cq = &vhci->cmd; + int do_post = 0; + + mtx_lock_spin(&cq->lock); + if (cq->pending != 0) { + uint16_t base_cmd; + + /* + * Accept only replies matching the expected command + * (with REPLY_FLAG and optionally CANCEL_FLAG). + * Drop stale replies from timed-out commands. + */ + base_cmd = msg->cmd & ~(BCE_VHCI_CMD_REPLY_FLAG | + BCE_VHCI_CMD_CANCEL_FLAG); + if (base_cmd == cq->expected_cmd) { + cq->response = *msg; + cq->pending = 0; + do_post = 1; + } + } + mtx_unlock_spin(&cq->lock); + + /* + * sema_post uses MTX_DEF internally; must not be called under + * MTX_SPIN + */ + if (do_post) + sema_post(&cq->completion); +} + +/* + * Handle port status change event from firmware (ISR context). + * Cannot call cmd_execute here (sleeps), so defer to taskqueue. + */ +static void +bce_vhci_handle_port_status_change(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg) +{ + uint32_t port; + + if (vhci->sc_detaching) + return; + + port = msg->param1; + if (port >= vhci->sc_port_count) + return; + + atomic_set_int(&vhci->sc_port_chg_mask, 1U << port); + taskqueue_enqueue(taskqueue_thread, &vhci->sc_port_chg_task); +} + +/* + * Deferred port status change handler (taskqueue context, can sleep). + * Queries firmware for current port status and updates the cache. + */ +static void +bce_vhci_port_chg_task(void *arg, int pending __unused) +{ + struct bce_vhci_softc *vhci = arg; + struct bce_vhci_message cmd, reply; + uint32_t mask, port, port_status; + int error; + + if (vhci->sc_detaching) + return; + + mask = atomic_readandclear_int(&vhci->sc_port_chg_mask); + + for (port = 0; mask != 0; port++, mask >>= 1) { + if ((mask & 1) == 0) + continue; + + device_printf(vhci->sc_dev, + "port %u status change\n", port); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_PORT_STATUS; + cmd.param1 = port; + + error = bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_SHORT); + + USB_BUS_LOCK(&vhci->sc_bus); + if (error == 0) { + port_status = (uint32_t)reply.param2; + + vhci->sc_port_status[port] = 0; + if (vhci->sc_port_power[port]) + vhci->sc_port_status[port] |= + UPS_PORT_POWER; + if (port_status & BCE_VHCI_PORT_ENABLED) + vhci->sc_port_status[port] |= + UPS_PORT_ENABLED | UPS_HIGH_SPEED; + if (port_status & BCE_VHCI_PORT_CONNECTED) + vhci->sc_port_status[port] |= + UPS_CURRENT_CONNECT_STATUS; + if (port_status & BCE_VHCI_PORT_SUSPENDED) + vhci->sc_port_status[port] |= + UPS_SUSPEND; + if (port_status & BCE_VHCI_PORT_OVERCURRENT) + vhci->sc_port_status[port] |= + UPS_OVERCURRENT_INDICATOR; + } + vhci->sc_port_change[port] |= UPS_C_CONNECT_STATUS; + USB_BUS_UNLOCK(&vhci->sc_bus); + } + + /* Wake the USB hub poll */ + usb_needs_explore(&vhci->sc_bus, 0); +} + +/* + * Deferred endpoint reset task. + * + * Called on taskqueue_thread (can sleep) after CTRL_TRANSFER_STATUS(STALL). + * Flushes residual SQ entries and issues ENDPOINT_RESET (0x0044) to clear + * firmware's stall state. + * + * After reset, clears tq->stalled so the USB stack's next retry succeeds. + */ +static void +bce_vhci_reset_one_tq(struct bce_vhci_softc *vhci, + struct bce_vhci_transfer_queue *tq) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + struct bce_vhci_message cmd, reply; + + device_printf(vhci->sc_dev, + "reset_task: flushing + ENDPOINT_RESET dev=%d ep=0x%02x\n", + tq->dev_addr, tq->endp_addr); + + /* Flush residual SQ submissions */ + if (tq->sq_in != NULL) + bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, tq->sq_in->qid); + if (tq->sq_out != NULL) + bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, tq->sq_out->qid); + + /* Issue ENDPOINT_RESET to clear firmware stall state */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_RESET; + cmd.param1 = tq->dev_addr | ((tq->endp_addr & 0x8F) << 8); + bce_vhci_cmd_execute(vhci, &cmd, &reply, BCE_VHCI_CMD_TIMEOUT_SHORT); + + device_printf(vhci->sc_dev, + "reset_task: ENDPOINT_RESET done, clearing stall\n"); + + USB_BUS_LOCK(&vhci->sc_bus); + tq->stalled = 0; + USB_BUS_UNLOCK(&vhci->sc_bus); +} + +static void +bce_vhci_reset_task(void *arg, int pending __unused) +{ + struct bce_vhci_softc *vhci = arg; + int i, j; + + for (i = 0; i < BCE_VHCI_MAX_DEVICES; i++) { + struct bce_vhci_device *dev = &vhci->sc_devs[i]; + + if (dev->allocated == 0) + continue; + for (j = 0; j < BCE_VHCI_MAX_ENDPOINTS; j++) { + struct bce_vhci_transfer_queue *tq = &dev->tq[j]; + + if (tq->active == 0 || tq->stalled == 0) + continue; + bce_vhci_reset_one_tq(vhci, tq); + } + } +} + +/* + * bce_vhci_create_task: deferred endpoint creation from taskqueue_thread. + * + * bce_vhci_pipe_start cannot call bce_vhci_endpoint_create directly because + * it may be invoked from a USB callback (e.g. usbhid_intr_in_callback) that + * holds a non-sleepable lock. Instead, pipe_start sets create_pending on the + * tq and schedules this task. We scan all devices/endpoints, create any with + * create_pending set, then return USB_ERR_STALLED from pipe_start so the USB + * stack retries, at which point tq->active is set and we skip creation. + */ +static void +bce_vhci_create_task(void *arg, int pending __unused) +{ + struct bce_vhci_softc *vhci = arg; + int i, j, ep_err; + + for (i = 0; i < BCE_VHCI_MAX_DEVICES; i++) { + struct bce_vhci_device *dev = &vhci->sc_devs[i]; + + if (dev->allocated == 0) + continue; + + for (j = 0; j < BCE_VHCI_MAX_ENDPOINTS; j++) { + struct bce_vhci_transfer_queue *tq = &dev->tq[j]; + struct usb_endpoint_descriptor *edesc; + struct usb_xfer *xfer; + uint8_t ep_addr; + + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->create_pending == 0 || tq->active) { + tq->create_pending = 0; + USB_BUS_UNLOCK(&vhci->sc_bus); + continue; + } + tq->create_pending = 0; + ep_addr = tq->endp_addr; + edesc = tq->create_edesc; + xfer = tq->create_xfer; + /* + * Do NOT clear create_xfer yet -- + * pipe_close may race + */ + USB_BUS_UNLOCK(&vhci->sc_bus); + + ep_err = bce_vhci_endpoint_create(vhci, dev, + ep_addr, edesc); + + /* + * Re-check create_xfer under lock. Atomically + * clear it and either install active_xfer or + * complete with error; no gap for pipe_close. + */ + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->create_xfer != xfer) { + /* + * Original xfer was closed; leave any + * newer create_xfer for pipe_start retry. + */ + USB_BUS_UNLOCK(&vhci->sc_bus); + continue; + } + + if (ep_err != 0) { + tq->create_xfer = NULL; + if (xfer != NULL) + usbd_transfer_done(xfer, + USB_ERR_STALLED); + USB_BUS_UNLOCK(&vhci->sc_bus); + device_printf(vhci->sc_dev, + "create_task: ep create " + "failed: dev=%d ep=0x%02x " + "err=%d\n", + dev->fw_dev_id, ep_addr, + ep_err); + continue; + } + + if (xfer != NULL && (ep_addr & UE_DIR_IN)) { + struct bce_vhci_message treq; + struct bce_qe_submission *si; + uint32_t len; + + len = xfer->frlengths[0]; + if (len > BCE_VHCI_XFER_BUFSZ) + len = BCE_VHCI_XFER_BUFSZ; + /* + * Handoff: clear create_xfer and + * set active_xfer atomically. + */ + tq->create_xfer = NULL; + tq->active_xfer = xfer; + tq->dma_inflight = 1; + USB_BUS_UNLOCK(&vhci->sc_bus); + + bus_dmamap_sync(tq->dma_tag, + tq->dma_map, + BUS_DMASYNC_PREREAD); + + /* Reserve msg first, then SQ */ + memset(&treq, 0, sizeof(treq)); + treq.cmd = + BCE_VHCI_CMD_TRANSFER_REQUEST; + treq.param1 = + ((uint32_t)ep_addr << 8) | + dev->fw_dev_id; + treq.param2 = len; + + mtx_lock_spin(&vhci->sc_async_lock); + if (bce_reserve_submission( + vhci->msg_asynchronous.sq) != 0) { + mtx_unlock_spin( + &vhci->sc_async_lock); + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + continue; + } + mtx_unlock_spin(&vhci->sc_async_lock); + /* active_xfer already set above */ + + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission( + tq->sq_in) == 0) { + si = bce_next_submission( + tq->sq_in); + si->addr = tq->dma_addr; + si->length = len; + si->segl_addr = 0; + si->segl_length = 0; + bce_submit_to_device( + vhci->sc_bce, + tq->sq_in); + mtx_unlock_spin(&tq->lock); + + mtx_lock_spin( + &vhci->sc_async_lock); + bce_vhci_msg_queue_write(vhci, + &vhci->msg_asynchronous, + &treq); + mtx_unlock_spin( + &vhci->sc_async_lock); + } else { + mtx_unlock_spin(&tq->lock); + /* Return reserved msg slot */ + mtx_lock_spin( + &vhci->sc_async_lock); + atomic_add_int(&vhci-> + msg_asynchronous.sq-> + available_commands, 1); + mtx_unlock_spin( + &vhci->sc_async_lock); + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + } + } else if (xfer != NULL && + (ep_addr & UE_DIR_IN) == 0) { + /* + * OUT endpoint: set active and submit. + */ + struct bce_vhci_message treq; + struct bce_qe_submission *so; + uint32_t len; + + len = xfer->frlengths[0]; + if (len > BCE_VHCI_XFER_BUFSZ) + len = BCE_VHCI_XFER_BUFSZ; + + tq->create_xfer = NULL; + tq->active_xfer = xfer; + tq->dma_inflight = 1; + + if (len > 0) { + usbd_copy_out( + &xfer->frbuffers[0], 0, + tq->dma_buf, len); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + + if (len > 0) { + bus_dmamap_sync(tq->dma_tag, + tq->dma_map, + BUS_DMASYNC_PREWRITE); + } + + memset(&treq, 0, sizeof(treq)); + treq.cmd = + BCE_VHCI_CMD_TRANSFER_REQUEST; + treq.param1 = + ((uint32_t)ep_addr << 8) | + dev->fw_dev_id; + treq.param2 = len; + + mtx_lock_spin(&vhci->sc_async_lock); + if (bce_reserve_submission( + vhci->msg_asynchronous.sq) != 0) { + mtx_unlock_spin( + &vhci->sc_async_lock); + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + continue; + } + mtx_unlock_spin(&vhci->sc_async_lock); + + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission( + tq->sq_out) == 0) { + so = bce_next_submission( + tq->sq_out); + so->addr = tq->dma_addr; + so->length = len; + so->segl_addr = 0; + so->segl_length = 0; + bce_submit_to_device( + vhci->sc_bce, + tq->sq_out); + mtx_unlock_spin(&tq->lock); + + mtx_lock_spin( + &vhci->sc_async_lock); + bce_vhci_msg_queue_write(vhci, + &vhci->msg_asynchronous, + &treq); + mtx_unlock_spin( + &vhci->sc_async_lock); + } else { + mtx_unlock_spin(&tq->lock); + mtx_lock_spin( + &vhci->sc_async_lock); + atomic_add_int(&vhci-> + msg_asynchronous.sq-> + available_commands, 1); + mtx_unlock_spin( + &vhci->sc_async_lock); + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + } + } else if (xfer != NULL) { + tq->create_xfer = NULL; + usbd_transfer_done(xfer, + USB_ERR_STALLED); + USB_BUS_UNLOCK(&vhci->sc_bus); + } else { + tq->create_xfer = NULL; + USB_BUS_UNLOCK(&vhci->sc_bus); + } + } + } +} + +/* + * Execute a synchronous command: send on msg_commands, wait for reply + * on ev_commands or ev_system. + */ +static int +bce_vhci_cmd_execute(struct bce_vhci_softc *vhci, + struct bce_vhci_message *req, struct bce_vhci_message *reply, + int timeout_ticks) +{ + struct bce_vhci_cmd_queue *cq = &vhci->cmd; + struct bce_vhci_message cancel; + int error; + + sx_xlock(&cq->exec_lock); + mtx_lock_spin(&cq->lock); + + /* Reserve a submission slot */ + if (bce_reserve_submission(cq->msg->sq) != 0) { + mtx_unlock_spin(&cq->lock); + sx_xunlock(&cq->exec_lock); + return (EAGAIN); + } + + /* Setup completion state */ + cq->pending = 1; + cq->expected_cmd = req->cmd; + memset(&cq->response, 0, sizeof(cq->response)); + + mtx_unlock_spin(&cq->lock); + + /* Send the command */ + bce_vhci_msg_queue_write(vhci, cq->msg, req); + + /* Wait for reply */ + error = sema_timedwait(&cq->completion, timeout_ticks); + + mtx_lock_spin(&cq->lock); + + if (error != 0) { + /* + * Timeout: send cancellation and wait briefly. + */ + device_printf(vhci->sc_dev, + "cmd 0x%04x timeout, sending cancel\n", req->cmd); + + if (bce_reserve_submission(cq->msg->sq) == 0) { + cancel = *req; + cancel.cmd |= BCE_VHCI_CMD_CANCEL_FLAG; + cq->pending = 1; + mtx_unlock_spin(&cq->lock); + + bce_vhci_msg_queue_write(vhci, cq->msg, &cancel); + + error = sema_timedwait(&cq->completion, hz); + + mtx_lock_spin(&cq->lock); + if (error != 0) { + device_printf(vhci->sc_dev, + "cmd cancel timeout, possible desync\n"); + cq->pending = 0; + mtx_unlock_spin(&cq->lock); + sx_xunlock(&cq->exec_lock); + return (ETIMEDOUT); + } + + /* + * Check if we got the cancel ack or the + * original reply + */ + if ((cq->response.cmd & ~BCE_VHCI_CMD_REPLY_FLAG) == + (req->cmd | BCE_VHCI_CMD_CANCEL_FLAG)) { + cq->pending = 0; + mtx_unlock_spin(&cq->lock); + sx_xunlock(&cq->exec_lock); + return (ETIMEDOUT); + } + /* Got original reply; fall through */ + } else { + cq->pending = 0; + mtx_unlock_spin(&cq->lock); + sx_xunlock(&cq->exec_lock); + return (ETIMEDOUT); + } + } + + /* Copy reply before releasing the lock */ + { + struct bce_vhci_message resp; + + resp = cq->response; + cq->pending = 0; + mtx_unlock_spin(&cq->lock); + sx_xunlock(&cq->exec_lock); + + if (reply != NULL) + *reply = resp; + + /* Validate reply from local copy */ + if ((resp.cmd & ~BCE_VHCI_CMD_REPLY_FLAG) != req->cmd) { + device_printf(vhci->sc_dev, + "cmd mismatch: sent 0x%04x, got 0x%04x\n", + req->cmd, resp.cmd); + return (EIO); + } + + if (resp.status != BCE_VHCI_SUCCESS) + return (resp.status); + } + + return (0); +} + +/* + * Submit a pending IN xfer after the previous one completed. + * Called under USB_BUS_LOCK. nxfer has been detached from + * tq->pending_xfer by the caller. + */ +static void +bce_vhci_submit_pending_in(struct bce_vhci_softc *vhci, + struct bce_vhci_transfer_queue *tq, struct usb_xfer *nxfer) +{ + struct bce_vhci_message treq; + struct bce_qe_submission *si; + uint32_t nlen; + + nlen = nxfer->frlengths[0]; + if (nlen > BCE_VHCI_XFER_BUFSZ) + nlen = BCE_VHCI_XFER_BUFSZ; + + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREREAD); + + memset(&treq, 0, sizeof(treq)); + treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + treq.param1 = + ((uint32_t)tq->endp_addr << 8) | tq->dev_addr; + treq.param2 = nlen; + + /* Reserve msg first, then SQ */ + mtx_lock_spin(&vhci->sc_async_lock); + if (bce_reserve_submission( + vhci->msg_asynchronous.sq) != 0) { + mtx_unlock_spin(&vhci->sc_async_lock); + usbd_transfer_done(nxfer, USB_ERR_IOERROR); + return; + } + mtx_unlock_spin(&vhci->sc_async_lock); + + /* + * Install active_xfer BEFORE ringing the doorbell. + * A fast completion could otherwise see NULL and + * discard the result. + */ + tq->active_xfer = nxfer; + tq->dma_inflight = 1; + + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_in) == 0) { + si = bce_next_submission(tq->sq_in); + si->addr = tq->dma_addr; + si->length = nlen; + si->segl_addr = 0; + si->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, tq->sq_in); + mtx_unlock_spin(&tq->lock); + + mtx_lock_spin(&vhci->sc_async_lock); + bce_vhci_msg_queue_write(vhci, + &vhci->msg_asynchronous, &treq); + mtx_unlock_spin(&vhci->sc_async_lock); + } else { + mtx_unlock_spin(&tq->lock); + tq->active_xfer = NULL; + tq->dma_inflight = 0; + /* Return reserved msg slot */ + mtx_lock_spin(&vhci->sc_async_lock); + atomic_add_int( + &vhci->msg_asynchronous.sq-> + available_commands, 1); + mtx_unlock_spin(&vhci->sc_async_lock); + usbd_transfer_done(nxfer, USB_ERR_IOERROR); + } +} + +/* + * Submit a pending OUT xfer after the previous one completed. + * Called under USB_BUS_LOCK. nxfer has been detached from + * tq->pending_xfer by the caller. + */ +static void +bce_vhci_submit_pending_out(struct bce_vhci_softc *vhci, + struct bce_vhci_transfer_queue *tq, struct usb_xfer *nxfer) +{ + struct bce_vhci_message treq; + struct bce_qe_submission *so; + uint32_t nlen; + + nlen = nxfer->frlengths[0]; + if (nlen > BCE_VHCI_XFER_BUFSZ) + nlen = BCE_VHCI_XFER_BUFSZ; + + usbd_copy_out(&nxfer->frbuffers[0], 0, + tq->dma_buf, nlen); + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREWRITE); + + memset(&treq, 0, sizeof(treq)); + treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + treq.param1 = + ((uint32_t)tq->endp_addr << 8) | tq->dev_addr; + treq.param2 = nlen; + + /* Reserve msg first, then SQ */ + mtx_lock_spin(&vhci->sc_async_lock); + if (bce_reserve_submission( + vhci->msg_asynchronous.sq) != 0) { + mtx_unlock_spin(&vhci->sc_async_lock); + usbd_transfer_done(nxfer, USB_ERR_IOERROR); + return; + } + mtx_unlock_spin(&vhci->sc_async_lock); + + /* + * Install active_xfer BEFORE ringing the doorbell. + */ + tq->active_xfer = nxfer; + tq->dma_inflight = 1; + + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_out) == 0) { + so = bce_next_submission(tq->sq_out); + so->addr = tq->dma_addr; + so->length = nlen; + so->segl_addr = 0; + so->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, tq->sq_out); + mtx_unlock_spin(&tq->lock); + + mtx_lock_spin(&vhci->sc_async_lock); + bce_vhci_msg_queue_write(vhci, + &vhci->msg_asynchronous, &treq); + mtx_unlock_spin(&vhci->sc_async_lock); + } else { + mtx_unlock_spin(&tq->lock); + tq->active_xfer = NULL; + tq->dma_inflight = 0; + /* Return reserved msg slot */ + mtx_lock_spin(&vhci->sc_async_lock); + atomic_add_int( + &vhci->msg_asynchronous.sq-> + available_commands, 1); + mtx_unlock_spin(&vhci->sc_async_lock); + usbd_transfer_done(nxfer, USB_ERR_IOERROR); + } +} + +/* + * Transfer queue DMA completion callback. Fires when the firmware + * has consumed (OUT) or filled (IN) a DMA buffer we submitted. + * + * For IN transfers, record the actual byte count from the completion + * so that handle_ctrl_status knows how much data was received. + */ +static void +bce_vhci_tq_completion(struct bce_queue_sq *sq) +{ + struct bce_vhci_transfer_queue *tq = sq->userdata; + struct bce_vhci_softc *vhci = tq->vhci; + + while (sq->completion_cidx != sq->completion_tail) { + struct bce_sq_completion_data *cd; + + cd = &sq->completion_data[sq->completion_cidx]; + + /* + * For IN SQ completions (device -> host), handle data. + * BCE uses ithreaded MSI, so we can acquire USB_BUS_LOCK + * (MTX_DEF) here. tq->lock (MTX_SPIN) nesting inside + * USB_BUS_LOCK is valid. + */ + if (sq == tq->sq_in && cd->status == BCE_COMP_SUCCESS) { + if (tq->endp_addr == 0x00) { + /* + * Control transfer: just record data length. + * Actual completion happens in + * handle_ctrl_status. Clamp to DMA buffer. + * USB_BUS_LOCK protects ctrl_actual and + * ctrl_data_done against concurrent access + * from handle_ctrl_status. + * + * If CTRL_TRANSFER_STATUS arrived first + * (ctrl_status_pending), process it now + * that data is ready. + */ + uint32_t alen = (uint32_t)cd->data_size; + if (alen > BCE_VHCI_XFER_BUFSZ) + alen = BCE_VHCI_XFER_BUFSZ; + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == NULL || + (tq->ctrl_state != BCE_VHCI_CTRL_STATUS && + tq->ctrl_state != BCE_VHCI_CTRL_DATA)) { + tq->dma_inflight = 0; + USB_BUS_UNLOCK(&vhci->sc_bus); + goto next_compl; + } + if (alen > tq->ctrl_data_len) + alen = tq->ctrl_data_len; + tq->ctrl_actual = alen; + tq->ctrl_data_done = 1; + if (tq->ctrl_status_pending != 0) { + tq->ctrl_status_pending = 0; + bce_vhci_complete_ctrl_locked( + vhci, tq, + &tq->ctrl_status_msg); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + } else { + /* + * Interrupt/bulk IN transfer: data is ready. + * Copy into xfer buffer and complete. + */ + struct usb_xfer *xfer; + uint32_t len = (uint32_t)cd->data_size; + + USB_BUS_LOCK(&vhci->sc_bus); + xfer = tq->active_xfer; + tq->dma_inflight = 0; + if (xfer != NULL) { + bus_dmamap_sync(tq->dma_tag, + tq->dma_map, + BUS_DMASYNC_POSTREAD); + + if (len > BCE_VHCI_XFER_BUFSZ) + len = BCE_VHCI_XFER_BUFSZ; + if (len > xfer->frlengths[0]) + len = xfer->frlengths[0]; + + usbd_copy_in(&xfer->frbuffers[0], 0, + tq->dma_buf, len); + xfer->frlengths[0] = len; + xfer->aframes = xfer->nframes; + tq->active_xfer = NULL; + + /* Start next queued xfer if any */ + if (tq->pending_xfer != NULL) { + struct usb_xfer *nxfer; + + nxfer = tq->pending_xfer; + tq->pending_xfer = NULL; + bce_vhci_submit_pending_in( + vhci, tq, nxfer); + } + + usbd_transfer_done(xfer, + USB_ERR_NORMAL_COMPLETION); + } else if (tq->pending_xfer != NULL) { + /* + * Stale completion from cancelled xfer. + * DMA drained; start pending xfer now. + */ + struct usb_xfer *nxfer; + + nxfer = tq->pending_xfer; + tq->pending_xfer = NULL; + bce_vhci_submit_pending_in( + vhci, tq, nxfer); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + } + } + + /* + * For ep0 OUT SQ completion in CTRL_SETUP state: + * setup packet DMA is done, start the data phase. + * USB_BUS_LOCK protects ctrl_state against concurrent + * access from pipe_start, pipe_close, and handle_ctrl_status. + */ + if (sq == tq->sq_out && tq->endp_addr == 0x00 && + cd->status == BCE_COMP_SUCCESS) { + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_POSTWRITE); + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == NULL) { + tq->dma_inflight = 0; + USB_BUS_UNLOCK(&vhci->sc_bus); + goto next_compl; + } + if (tq->ctrl_state == BCE_VHCI_CTRL_SETUP) { + if (tq->ctrl_data_len > 0) { + tq->ctrl_state = BCE_VHCI_CTRL_DATA; + } else { + tq->ctrl_state = BCE_VHCI_CTRL_STATUS; + } + /* + * CTRL_TRANSFER_STATUS may have arrived + * before setup DMA completed. Process + * the deferred status now. + */ + if (tq->ctrl_status_pending != 0) { + tq->ctrl_status_pending = 0; + bce_vhci_complete_ctrl_locked( + vhci, tq, + &tq->ctrl_status_msg); + } + } else if (tq->ctrl_state == + BCE_VHCI_CTRL_STATUS && + tq->ctrl_dir == UE_DIR_OUT) { + /* + * OUT data DMA done. Allow + * CTRL_TRANSFER_STATUS to proceed. + */ + tq->ctrl_data_done = 1; + if (tq->ctrl_status_pending != 0) { + tq->ctrl_status_pending = 0; + bce_vhci_complete_ctrl_locked( + vhci, tq, + &tq->ctrl_status_msg); + } + } + USB_BUS_UNLOCK(&vhci->sc_bus); + /* + * Data phase (both IN and OUT) is driven by firmware + * TRANSFER_REQUEST events handled in + * handle_transfer_request(). + */ + } + + /* + * For OUT SQ completions on non-control endpoints, + * the firmware consumed our data; complete the xfer. + */ + if (sq == tq->sq_out && tq->endp_addr != 0x00 && + cd->status == BCE_COMP_SUCCESS) { + struct usb_xfer *xfer; + + /* + * POSTWRITE before CPU touches buffer again. + */ + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_POSTWRITE); + + USB_BUS_LOCK(&vhci->sc_bus); + xfer = tq->active_xfer; + tq->dma_inflight = 0; + if (xfer != NULL) { + tq->active_xfer = NULL; + + /* Start next queued OUT xfer if any */ + if (tq->pending_xfer != NULL) { + struct usb_xfer *nxfer; + + nxfer = tq->pending_xfer; + tq->pending_xfer = NULL; + bce_vhci_submit_pending_out( + vhci, tq, nxfer); + } + + xfer->aframes = xfer->nframes; + usbd_transfer_done(xfer, + USB_ERR_NORMAL_COMPLETION); + } else if (tq->pending_xfer != NULL) { + struct usb_xfer *nxfer; + + nxfer = tq->pending_xfer; + tq->pending_xfer = NULL; + bce_vhci_submit_pending_out( + vhci, tq, nxfer); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + } + + /* + * Handle SQ error completions. Clear dma_inflight + * and complete active xfer with error so the endpoint + * is not permanently stuck. + */ + if (cd->status != BCE_COMP_SUCCESS) { + struct usb_xfer *xfer, *pxfer; + + USB_BUS_LOCK(&vhci->sc_bus); + xfer = tq->active_xfer; + pxfer = tq->pending_xfer; + tq->dma_inflight = 0; + if (xfer != NULL) { + tq->active_xfer = NULL; + tq->pending_xfer = NULL; + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + if (pxfer != NULL) + usbd_transfer_done(pxfer, + USB_ERR_IOERROR); + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } else if (pxfer != NULL) { + tq->pending_xfer = NULL; + usbd_transfer_done(pxfer, + USB_ERR_IOERROR); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + } + +next_compl: + sq->completion_cidx = + (sq->completion_cidx + 1) % sq->el_count; + bce_notify_submission_complete(sq); + } +} + +/* + * Create per-endpoint DMA transfer queues and register with firmware. + */ +static int +bce_vhci_endpoint_create(struct bce_vhci_softc *vhci, + struct bce_vhci_device *dev, uint8_t ep_addr, + struct usb_endpoint_descriptor *edesc) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + struct bce_vhci_transfer_queue *tq; + struct bce_queue_memcfg cfg; + struct bce_vhci_dma_cb_arg cb; + struct bce_vhci_message cmd, reply; + char name[0x20]; + uint32_t status; + int error, cq_qid, out_qid, in_qid, i; + uint8_t ep_idx; + + ep_idx = bce_vhci_ep_index(ep_addr); + if (ep_idx >= BCE_VHCI_MAX_ENDPOINTS) + return (EINVAL); + + tq = &dev->tq[ep_idx]; + if (tq->active) + return (EEXIST); + + /* + * Initialize runtime fields. Do NOT zero the whole struct: + * create_xfer/create_pending are live state managed by + * create_task under USB_BUS_LOCK. + */ + tq->vhci = vhci; + tq->dev_addr = dev->fw_dev_id; + tq->endp_addr = ep_addr; + tq->cq = NULL; + tq->sq_in = NULL; + tq->sq_out = NULL; + tq->active_xfer = NULL; + tq->pending_xfer = NULL; + tq->paused_by = 0; + tq->active = 0; + tq->stalled = 0; + tq->dma_inflight = 0; + + /* Free leftover DMA buffer from previous incarnation */ + if (tq->dma_tag != NULL) { + bus_dmamap_unload(tq->dma_tag, tq->dma_map); + bus_dmamem_free(tq->dma_tag, tq->dma_buf, tq->dma_map); + bus_dma_tag_destroy(tq->dma_tag); + tq->dma_tag = NULL; + } + tq->dma_map = NULL; + tq->dma_addr = 0; + tq->dma_buf = NULL; + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + tq->ctrl_dir = 0; + tq->ctrl_data_len = 0; + tq->ctrl_actual = 0; + tq->ctrl_data_done = 0; + tq->ctrl_status_pending = 0; + tq->evt_pending = 0; + /* tq->lock initialized in device_create, valid for device lifetime */ + + /* Allocate DMA buffer for data transfers */ + error = bus_dma_tag_create(sc->sc_dma_tag, + 4, 0, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, + NULL, NULL, + BCE_VHCI_XFER_BUFSZ, 1, BCE_VHCI_XFER_BUFSZ, + BUS_DMA_WAITOK, + NULL, NULL, + &tq->dma_tag); + if (error != 0) + return (error); + + error = bus_dmamem_alloc(tq->dma_tag, &tq->dma_buf, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, + &tq->dma_map); + if (error != 0) + goto fail_tag; + + error = bus_dmamap_load(tq->dma_tag, tq->dma_map, tq->dma_buf, + BCE_VHCI_XFER_BUFSZ, bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK); + if (error != 0 || cb.error != 0) { + error = error != 0 ? error : cb.error; + goto fail_mem; + } + tq->dma_addr = cb.addr; + + /* Allocate CQ for this endpoint */ + cq_qid = bce_vhci_alloc_qid(vhci); + if (cq_qid < 0) { + error = ENOSPC; + goto fail_dma; + } + tq->cq = bce_alloc_cq(sc, cq_qid, BCE_VHCI_TQ_EL); + if (tq->cq == NULL) { + error = ENOMEM; + goto fail_cq_alloc; + } + + bce_get_cq_memcfg(tq->cq, &cfg); + cfg.vector_or_cq = 4; + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, NULL, 0); + if (status != 0) { + error = EIO; + goto fail_cq; + } + + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[cq_qid] = tq->cq; + { + int inserted = 0; + + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == NULL) { + sc->sc_cq_list[i] = tq->cq; + inserted = 1; + break; + } + } + if (inserted == 0) { + sc->sc_queues[cq_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + device_printf(vhci->sc_dev, + "CQ list full, cannot add endpoint CQ\n"); + error = ENOSPC; + goto fail_cq_reg; + } + } + mtx_unlock(&sc->sc_queues_lock); + + /* Allocate OUT SQ (host -> device) */ + out_qid = bce_vhci_alloc_qid(vhci); + if (out_qid < 0) { + error = ENOSPC; + goto fail_cq_reg; + } + tq->sq_out = bce_alloc_sq(sc, out_qid, + sizeof(struct bce_qe_submission), BCE_VHCI_TQ_EL, + bce_vhci_tq_completion, tq); + if (tq->sq_out == NULL) { + error = ENOMEM; + goto fail_sq_out_alloc; + } + + snprintf(name, sizeof(name), "VHC1-%d-%02x", + dev->fw_dev_id, ep_addr & 0x0F); + bce_get_sq_memcfg(tq->sq_out, tq->cq, &cfg); + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 1); + if (status != 0) { + device_printf(vhci->sc_dev, + "failed to register OUT SQ '%s': %u\n", name, status); + error = EIO; + goto fail_sq_out; + } + + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[out_qid] = tq->sq_out; + sc->sc_int_sq_list[out_qid] = tq->sq_out; + mtx_unlock(&sc->sc_queues_lock); + + /* Allocate IN SQ (device -> host) */ + in_qid = bce_vhci_alloc_qid(vhci); + if (in_qid < 0) { + error = ENOSPC; + goto fail_sq_out_reg; + } + tq->sq_in = bce_alloc_sq(sc, in_qid, + sizeof(struct bce_qe_submission), BCE_VHCI_TQ_EL, + bce_vhci_tq_completion, tq); + if (tq->sq_in == NULL) { + error = ENOMEM; + goto fail_sq_in_alloc; + } + + snprintf(name, sizeof(name), "VHC1-%d-%02x", + dev->fw_dev_id, ep_addr | 0x80); + bce_get_sq_memcfg(tq->sq_in, tq->cq, &cfg); + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 0); + if (status != 0) { + device_printf(vhci->sc_dev, + "failed to register IN SQ '%s': %u\n", name, status); + error = EIO; + goto fail_sq_in; + } + + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[in_qid] = tq->sq_in; + sc->sc_int_sq_list[in_qid] = tq->sq_in; + mtx_unlock(&sc->sc_queues_lock); + + /* Tell firmware to create the endpoint */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_CREATE; + /* + * param1 = dev_id | ((ep_addr & 0x8F) << 8) + * param2 = type | (interval<<8) | (maxp<<16) | (maxp_burst<<32) + * Fields encode type, interval, maxpacket, and burst. + */ + cmd.param1 = dev->fw_dev_id | + ((uint32_t)(ep_addr & 0x8F) << 8); + if (edesc != NULL) { + uint8_t ep_type = UE_GET_XFERTYPE(edesc->bmAttributes); + uint16_t maxp = UGETW(edesc->wMaxPacketSize) & 0x7FF; + uint8_t mult = ((UGETW(edesc->wMaxPacketSize) >> 11) & 3) + 1; + uint64_t maxp_burst = (uint64_t)mult * maxp; + + cmd.param2 = ep_type; + if (ep_type == UE_INTERRUPT || ep_type == UE_ISOCHRONOUS) + cmd.param2 |= (uint64_t)(edesc->bInterval - 1) << 8; + cmd.param2 |= (uint64_t)maxp << 16; + cmd.param2 |= maxp_burst << 32; + } + /* ep0: edesc=NULL -> param2=0, firmware uses defaults for control */ + + error = bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_SHORT); + if (error != 0) { + device_printf(vhci->sc_dev, + "ENDPOINT_CREATE(dev=%d, ep=0x%02x) failed: %d\n", + dev->fw_dev_id, ep_addr, error); + goto fail_sq_in_reg; + } + + tq->active = 1; + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + + device_printf(vhci->sc_dev, + "endpoint created: dev=%d ep=0x%02x\n", + dev->fw_dev_id, ep_addr); + + return (0); + +fail_sq_in_reg: + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, in_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[in_qid] = NULL; + sc->sc_int_sq_list[in_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); +fail_sq_in: + bce_free_sq(sc, tq->sq_in); + tq->sq_in = NULL; +fail_sq_in_alloc: + bce_vhci_free_qid(vhci, in_qid); +fail_sq_out_reg: + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, out_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[out_qid] = NULL; + sc->sc_int_sq_list[out_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); +fail_sq_out: + bce_free_sq(sc, tq->sq_out); + tq->sq_out = NULL; +fail_sq_out_alloc: + bce_vhci_free_qid(vhci, out_qid); +fail_cq_reg: + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, cq_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[cq_qid] = NULL; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == tq->cq) { + sc->sc_cq_list[i] = NULL; + break; + } + } + mtx_unlock(&sc->sc_queues_lock); +fail_cq: + bce_free_cq(sc, tq->cq); + tq->cq = NULL; +fail_cq_alloc: + bce_vhci_free_qid(vhci, cq_qid); +fail_dma: + bus_dmamap_unload(tq->dma_tag, tq->dma_map); +fail_mem: + bus_dmamem_free(tq->dma_tag, tq->dma_buf, tq->dma_map); +fail_tag: + bus_dma_tag_destroy(tq->dma_tag); + tq->dma_tag = NULL; + return (error); +} + +/* + * Destroy a per-endpoint transfer queue. + */ +static void +bce_vhci_endpoint_destroy(struct bce_vhci_softc *vhci, + struct bce_vhci_device *dev, uint8_t ep_addr) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + struct bce_vhci_transfer_queue *tq; + struct bce_vhci_message cmd, reply; + uint8_t ep_idx; + int i; + + ep_idx = bce_vhci_ep_index(ep_addr); + if (ep_idx >= BCE_VHCI_MAX_ENDPOINTS) + return; + + tq = &dev->tq[ep_idx]; + if (tq->active == 0) + return; + + /* + * Mark inactive and complete any orphaned transfers under USB_BUS_LOCK. + * IRQ event handlers (find_tq) check tq->active under USB_BUS_LOCK, + * so clearing it here prevents concurrent access during teardown. + */ + USB_BUS_LOCK(&vhci->sc_bus); + tq->active = 0; + tq->dma_inflight = 0; + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + { + struct usb_xfer *ax, *px, *cx; + + ax = tq->active_xfer; + px = tq->pending_xfer; + cx = tq->create_xfer; + tq->active_xfer = NULL; + tq->pending_xfer = NULL; + tq->create_xfer = NULL; + tq->create_pending = 0; + + if (ax != NULL) + usbd_transfer_done(ax, USB_ERR_CANCELLED); + if (px != NULL) + usbd_transfer_done(px, USB_ERR_CANCELLED); + if (cx != NULL) + usbd_transfer_done(cx, USB_ERR_CANCELLED); + } + USB_BUS_UNLOCK(&vhci->sc_bus); + + /* + * Drain the reset task to ensure it is not accessing this tq's + * queues concurrently. Must be done without USB_BUS_LOCK held + * (taskqueue_drain may sleep). + */ + taskqueue_drain(taskqueue_thread, &vhci->sc_reset_task); + + /* Tell firmware to destroy the endpoint */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_DESTROY; + /* param1 = dev_id | ((ep_addr & 0x8F) << 8) */ + cmd.param1 = dev->fw_dev_id | + ((uint32_t)(ep_addr & 0x8F) << 8); + bce_vhci_cmd_execute(vhci, &cmd, &reply, BCE_VHCI_CMD_TIMEOUT_SHORT); + + /* Tear down IN SQ */ + if (tq->sq_in != NULL) { + int in_qid = tq->sq_in->qid; + + bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, in_qid); + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, in_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[in_qid] = NULL; + sc->sc_int_sq_list[in_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + bce_free_sq(sc, tq->sq_in); + tq->sq_in = NULL; + bce_vhci_free_qid(vhci, in_qid); + } + + /* Tear down OUT SQ */ + if (tq->sq_out != NULL) { + int out_qid = tq->sq_out->qid; + + bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, out_qid); + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, out_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[out_qid] = NULL; + sc->sc_int_sq_list[out_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + bce_free_sq(sc, tq->sq_out); + tq->sq_out = NULL; + bce_vhci_free_qid(vhci, out_qid); + } + + /* Tear down CQ */ + if (tq->cq != NULL) { + int cq_qid = tq->cq->qid; + + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, cq_qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[cq_qid] = NULL; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == tq->cq) { + sc->sc_cq_list[i] = NULL; + break; + } + } + mtx_unlock(&sc->sc_queues_lock); + bce_free_cq(sc, tq->cq); + tq->cq = NULL; + bce_vhci_free_qid(vhci, cq_qid); + } + + /* + * Keep DMA buffer alive: an ISR handler on a different event + * SQ may have passed find_tq before we unregistered the CQ + * and still references tq->dma_tag/dma_addr. The buffer is + * freed in bce_vhci_tq_destroy (device_destroy / detach). + */ + + /* tq->lock stays valid until device_destroy */ + + device_printf(vhci->sc_dev, + "endpoint destroyed: dev=%d ep=0x%02x\n", + dev->fw_dev_id, ep_addr); +} + +/* + * Create a firmware device on a port and set up ep0 queues. + * Called from the roothub SetPortFeature(PORT_RESET) path. + * + * NOTE: This runs from process context (USB explore thread) so it is + * safe to sleep in bce_vhci_cmd_execute. + */ +static int +bce_vhci_device_create(struct bce_vhci_softc *vhci, uint8_t port) +{ + struct bce_vhci_message cmd, reply; + struct bce_vhci_device *dev; + uint8_t fw_dev_id; + int error, i; + + /* Port reset */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_PORT_RESET; + cmd.param1 = port; + cmd.param2 = 1000; /* timeout ms */ + + error = bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_LONG); + if (error != 0) { + device_printf(vhci->sc_dev, + "PORT_RESET(%d) failed: %d\n", port, error); + return (error); + } + + device_printf(vhci->sc_dev, "port %d reset complete\n", port); + + /* Create device */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_DEVICE_CREATE; + cmd.param1 = port; + + error = bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_SHORT); + if (error != 0) { + device_printf(vhci->sc_dev, + "DEVICE_CREATE(port=%d) failed: %d\n", port, error); + return (error); + } + + if (reply.param2 >= BCE_VHCI_MAX_DEVICES) { + device_printf(vhci->sc_dev, + "firmware device ID %llu out of range\n", + (unsigned long long)reply.param2); + /* Destroy the firmware device we just created */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY; + cmd.param1 = (uint32_t)reply.param2; + bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_SHORT); + return (ERANGE); + } + + fw_dev_id = (uint8_t)reply.param2; + device_printf(vhci->sc_dev, + "device created: port=%d fw_dev_id=%d\n", port, fw_dev_id); + + dev = &vhci->sc_devs[fw_dev_id]; + memset(dev, 0, sizeof(*dev)); + dev->allocated = 1; + dev->fw_dev_id = fw_dev_id; + dev->port = port; + vhci->sc_port_to_dev[port] = fw_dev_id; + + /* Initialize per-endpoint locks (valid for device lifetime) */ + { + int i; + + for (i = 0; i < BCE_VHCI_MAX_ENDPOINTS; i++) + mtx_init(&dev->tq[i].lock, "bce_vhci_tq", + NULL, MTX_SPIN); + } + + /* Create ep0 (control endpoint, edesc=NULL -> firmware defaults) */ + error = bce_vhci_endpoint_create(vhci, dev, 0x00, NULL); + if (error != 0) { + device_printf(vhci->sc_dev, + "failed to create ep0 for dev %d: %d\n", + fw_dev_id, error); + /* Destroy the device */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY; + cmd.param1 = fw_dev_id; + bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_SHORT); + dev->allocated = 0; + for (i = 0; i < BCE_VHCI_MAX_ENDPOINTS; i++) + mtx_destroy(&dev->tq[i].lock); + vhci->sc_port_to_dev[port] = 0xFF; + return (error); + } + + return (0); +} + +/* + * Destroy a firmware device and all its endpoints. + */ +static void +bce_vhci_device_destroy(struct bce_vhci_softc *vhci, uint8_t port) +{ + struct bce_vhci_device *dev; + struct bce_vhci_message cmd, reply; + uint8_t fw_dev_id; + int i; + + if (port >= BCE_VHCI_MAX_PORTS) + return; + + fw_dev_id = vhci->sc_port_to_dev[port]; + if (fw_dev_id >= BCE_VHCI_MAX_DEVICES) + return; + + dev = &vhci->sc_devs[fw_dev_id]; + if (dev->allocated == 0) + return; + + /* + * Drain the create task so it does not race endpoint creation + * against our teardown. Must be done before destroying + * endpoints (taskqueue_drain may sleep). + */ + taskqueue_drain(taskqueue_thread, &vhci->sc_create_task); + taskqueue_drain(taskqueue_thread, &vhci->sc_fwevt_task); + + /* Destroy all active endpoints */ + for (i = 0; i < BCE_VHCI_MAX_ENDPOINTS; i++) { + if (dev->tq[i].active) + bce_vhci_endpoint_destroy(vhci, dev, + dev->tq[i].endp_addr); + } + + /* Destroy the firmware device */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY; + cmd.param1 = fw_dev_id; + bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_SHORT); + + /* Free deferred DMA buffers and destroy per-endpoint locks */ + for (i = 0; i < BCE_VHCI_MAX_ENDPOINTS; i++) { + struct bce_vhci_transfer_queue *tq = &dev->tq[i]; + + if (tq->dma_tag != NULL) { + bus_dmamap_unload(tq->dma_tag, tq->dma_map); + bus_dmamem_free(tq->dma_tag, tq->dma_buf, + tq->dma_map); + bus_dma_tag_destroy(tq->dma_tag); + tq->dma_tag = NULL; + } + mtx_destroy(&tq->lock); + } + + dev->allocated = 0; + vhci->sc_port_to_dev[port] = 0xFF; + + device_printf(vhci->sc_dev, + "device destroyed: port=%d fw_dev_id=%d\n", + port, fw_dev_id); +} + +/* + * Find the transfer queue for a given firmware device ID and endpoint. + * + * No USB_BUS_LOCK needed: endpoint_destroy clears tq->active under + * USB_BUS_LOCK first (preventing new find_tq matches), then sends + * ENDPOINT_DESTROY synchronously, then frees SQ/DMA resources. + * Callers that drop USB_BUS_LOCK before SQ operations recheck + * tq->active to handle the narrow window between active=0 and + * resource free. dev->allocated and tq->active are int-aligned; + * reads are safe on x86 (aligned word reads are atomic). + */ +static struct bce_vhci_transfer_queue * +bce_vhci_find_tq(struct bce_vhci_softc *vhci, uint8_t dev_id, uint8_t ep_addr) +{ + struct bce_vhci_device *dev; + uint8_t ep_idx; + + if (dev_id >= BCE_VHCI_MAX_DEVICES) + return (NULL); + + dev = &vhci->sc_devs[dev_id]; + if (dev->allocated == 0) + return (NULL); + + ep_idx = bce_vhci_ep_index(ep_addr); + if (ep_idx >= BCE_VHCI_MAX_ENDPOINTS) + return (NULL); + + if (dev->tq[ep_idx].active == 0) + return (NULL); + + return (&dev->tq[ep_idx]); +} + +/* + * Handle TRANSFER_REQUEST from firmware. + * + * The firmware asks us for data by sending TRANSFER_REQUEST with: + * param1 = (ep_addr << 8) | dev_id + * param2 = requested byte count + * + * For control transfers this drives the setup/data phases. + */ +static void +bce_vhci_handle_transfer_request(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg) +{ + struct bce_vhci_transfer_queue *tq; + struct usb_xfer *xfer; + struct bce_qe_submission *s; + uint8_t dev_id, ep_addr; + uint32_t req_len; + int bus_locked; + + dev_id = msg->param1 & 0xFF; + ep_addr = (msg->param1 >> 8) & 0xFF; + req_len = (uint32_t)msg->param2; + + tq = bce_vhci_find_tq(vhci, dev_id, ep_addr); + if (tq == NULL) { + device_printf(vhci->sc_dev, + "TRANSFER_REQUEST for unknown dev=%d ep=0x%02x\n", + dev_id, ep_addr); + return; + } + + /* + * Read active_xfer under USB_BUS_LOCK to serialize with pipe_close. + * Called from ev_generic_completion (ithread) or from pipe_start + * (already under USB_BUS_LOCK) via evt_pending replay. + */ + bus_locked = mtx_owned(&vhci->sc_bus.bus_mtx); + if (bus_locked == 0) + USB_BUS_LOCK(&vhci->sc_bus); + xfer = tq->active_xfer; + if (xfer == NULL) { + /* + * Firmware sends TRANSFER_REQUEST before the USB stack + * submits the xfer via pipe_start. Save the event and + * replay it when pipe_start fires. + */ + tq->evt_pending = 1; + tq->evt_saved = *msg; + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + return; + } + /* + * For non-control endpoints (interrupt/bulk), firmware is + * requesting or providing data. Submit the appropriate buffer. + */ + if (tq->endp_addr != 0x00) { + uint32_t len = req_len; + + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + + if (len > BCE_VHCI_XFER_BUFSZ) + len = BCE_VHCI_XFER_BUFSZ; + + /* + * Re-check tq->active after dropping USB_BUS_LOCK. + * endpoint_destroy sets active=0 under the lock before + * freeing SQ/DMA resources, so if it is clear, our + * SQ pointers may be stale. + */ + if (tq->active == 0) + return; + + if (ep_addr & 0x80) { + /* IN: firmware has data for us, submit receive buf */ + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREREAD); + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_in) == 0) { + s = bce_next_submission(tq->sq_in); + s->addr = tq->dma_addr; + s->length = len; + s->segl_addr = 0; + s->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, + tq->sq_in); + mtx_unlock_spin(&tq->lock); + } else { + mtx_unlock_spin(&tq->lock); + /* SQ full; fail the transfer */ + if (bus_locked == 0) + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + } + } else { + /* OUT: firmware wants data from us */ + if (bus_locked == 0) + USB_BUS_LOCK(&vhci->sc_bus); + if (xfer == tq->active_xfer && + xfer->frlengths[0] > 0) { + if (len > xfer->frlengths[0]) + len = xfer->frlengths[0]; + usbd_copy_out(&xfer->frbuffers[0], 0, + tq->dma_buf, len); + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREWRITE); + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_out) == 0) { + s = bce_next_submission(tq->sq_out); + s->addr = tq->dma_addr; + s->length = len; + s->segl_addr = 0; + s->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, + tq->sq_out); + mtx_unlock_spin(&tq->lock); + } else { + mtx_unlock_spin(&tq->lock); + /* SQ full; fail the transfer */ + if (bus_locked == 0) + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + } + } else { + /* + * Zero-length OUT or cancelled xfer. + * Complete immediately, then start + * any pending transfer. + */ + if (xfer == tq->active_xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + xfer->aframes = xfer->nframes; + + if (tq->pending_xfer != NULL) { + struct usb_xfer *nx; + + nx = tq->pending_xfer; + tq->pending_xfer = NULL; + bce_vhci_submit_pending_out( + vhci, tq, nx); + } + + usbd_transfer_done(xfer, + USB_ERR_NORMAL_COMPLETION); + } + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + } + } + return; + } + + /* + * Control endpoint (ep0) state machine. + * USB_BUS_LOCK is held here, protecting tq->active_xfer, + * tq->ctrl_state, and xfer validity. We drop and re-validate + * only around spin-lock + DMA submission sections. + */ + if (tq->active_xfer != xfer) { + /* Transfer was cancelled while we set up; bail */ + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + return; + } + + tq->dma_inflight = 1; + + switch (tq->ctrl_state) { + case BCE_VHCI_CTRL_SETUP: + { + /* + * Firmware wants the 8-byte setup packet. + * Copy from xfer frbuffers[0] into DMA buffer and submit + * on the OUT SQ. + */ + uint32_t len; + + len = req_len; + if (len > 8) + len = 8; + if (len > BCE_VHCI_XFER_BUFSZ) + len = BCE_VHCI_XFER_BUFSZ; + + usbd_copy_out(&xfer->frbuffers[0], 0, tq->dma_buf, len); + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + + if (tq->active == 0) + return; + + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREWRITE); + + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_out) != 0) { + mtx_unlock_spin(&tq->lock); + device_printf(vhci->sc_dev, + "no OUT SQ slot for setup\n"); + if (bus_locked == 0) + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + usbd_transfer_done(xfer, USB_ERR_IOERROR); + } + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + return; + } + + s = bce_next_submission(tq->sq_out); + s->addr = tq->dma_addr; + s->length = len; + s->segl_addr = 0; + s->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, tq->sq_out); + mtx_unlock_spin(&tq->lock); + + /* + * Wait for OUT SQ completion (setup DMA done) before + * starting the data phase. tq_completion will see + * CTRL_SETUP state on ep0 OUT completion and call + * data_start. Stay in CTRL_SETUP until then. + */ + break; + } + + case BCE_VHCI_CTRL_DATA: + { + /* + * Data phase. Direction was determined from the setup + * packet bmRequestType bit 7. + * USB_BUS_LOCK is held on entry (protects xfer, ctrl_state). + */ + uint32_t len; + + len = req_len; + if (len > BCE_VHCI_XFER_BUFSZ) + len = BCE_VHCI_XFER_BUFSZ; + if (len > tq->ctrl_data_len) + len = tq->ctrl_data_len; + + /* + * Transition to STATUS before submitting DMA so that + * the SQ completion handler sees the correct state. + */ + tq->ctrl_state = BCE_VHCI_CTRL_STATUS; + if (tq->ctrl_dir == UE_DIR_OUT) + tq->ctrl_actual = len; + + if (tq->ctrl_dir == UE_DIR_IN) { + /* + * Device -> host: reserve msg_asynchronous FIRST, + * then submit receive buffer on IN SQ. + * Correct ordering prevents an orphaned SQ entry + * if the msg slot is exhausted. + */ + struct bce_vhci_message treq; + + memset(&treq, 0, sizeof(treq)); + treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + treq.param1 = + ((uint32_t)tq->endp_addr << 8) | tq->dev_addr; + treq.param2 = len; + + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + + if (tq->active == 0) + return; + + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREREAD); + + /* Reserve msg slot first */ + mtx_lock_spin(&vhci->sc_async_lock); + if (bce_reserve_submission( + vhci->msg_asynchronous.sq) != 0) { + mtx_unlock_spin(&vhci->sc_async_lock); + device_printf(vhci->sc_dev, + "no msg_async slot for " + "ctrl data IN\n"); + if (bus_locked == 0) + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + tq->ctrl_state = + BCE_VHCI_CTRL_IDLE; + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + return; + } + mtx_unlock_spin(&vhci->sc_async_lock); + + /* Now reserve and submit IN SQ */ + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_in) != 0) { + mtx_unlock_spin(&tq->lock); + /* Return the reserved msg slot */ + mtx_lock_spin(&vhci->sc_async_lock); + atomic_add_int(&vhci-> + msg_asynchronous.sq-> + available_commands, 1); + mtx_unlock_spin(&vhci->sc_async_lock); + device_printf(vhci->sc_dev, + "no IN SQ slot for ctrl data\n"); + if (bus_locked == 0) + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + tq->ctrl_state = + BCE_VHCI_CTRL_IDLE; + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + return; + } + s = bce_next_submission(tq->sq_in); + s->addr = tq->dma_addr; + s->length = len; + s->segl_addr = 0; + s->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, tq->sq_in); + mtx_unlock_spin(&tq->lock); + + mtx_lock_spin(&vhci->sc_async_lock); + bce_vhci_msg_queue_write(vhci, + &vhci->msg_asynchronous, &treq); + mtx_unlock_spin(&vhci->sc_async_lock); + } else { + /* + * Host -> device: copy data from xfer frbuffers[1] + * and submit on OUT SQ. usbd_copy_out under + * USB_BUS_LOCK protects xfer validity. + */ + usbd_copy_out(&xfer->frbuffers[1], 0, + tq->dma_buf, len); + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + + if (tq->active == 0) + return; + + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREWRITE); + + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_out) != 0) { + mtx_unlock_spin(&tq->lock); + device_printf(vhci->sc_dev, + "no OUT SQ slot for data\n"); + if (bus_locked == 0) + USB_BUS_LOCK(&vhci->sc_bus); + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->dma_inflight = 0; + tq->ctrl_state = + BCE_VHCI_CTRL_IDLE; + usbd_transfer_done(xfer, + USB_ERR_IOERROR); + } + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + return; + } + + s = bce_next_submission(tq->sq_out); + s->addr = tq->dma_addr; + s->length = len; + s->segl_addr = 0; + s->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, tq->sq_out); + mtx_unlock_spin(&tq->lock); + } + + break; + } + + default: + device_printf(vhci->sc_dev, + "unexpected TRANSFER_REQUEST in state %d\n", + tq->ctrl_state); + tq->dma_inflight = 0; + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + usbd_transfer_done(xfer, USB_ERR_IOERROR); + } + if (bus_locked == 0) + USB_BUS_UNLOCK(&vhci->sc_bus); + break; + } +} + +/* + * Complete a control transfer. Caller must hold USB_BUS_LOCK. + * Maps firmware status to USB error and calls usbd_transfer_done. + */ +static void +bce_vhci_complete_ctrl_locked(struct bce_vhci_softc *vhci, + struct bce_vhci_transfer_queue *tq, struct bce_vhci_message *msg) +{ + struct usb_xfer *xfer; + usb_error_t usb_err; + + xfer = tq->active_xfer; + if (xfer == NULL) + return; + + /* Map firmware status to USB error */ + switch (msg->status) { + case BCE_VHCI_SUCCESS: + usb_err = USB_ERR_NORMAL_COMPLETION; + + /* + * Tell the USB stack all frames completed. + * usbd_transfer_done computes + * actlen = sum(frlengths[0..aframes-1]). + * If aframes stays 0, actlen=0 < sumlen -> USB_ERR_SHORT_XFER. + */ + xfer->aframes = xfer->nframes; + + /* + * If this was an IN data transfer, copy the received + * data back into the xfer buffer using usbd_copy_in + * (correct API for page-cache buffers). + */ + if (tq->ctrl_dir == UE_DIR_IN && tq->ctrl_actual > 0) { + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_POSTREAD); + + usbd_copy_in(&xfer->frbuffers[1], 0, + tq->dma_buf, tq->ctrl_actual); + xfer->frlengths[1] = tq->ctrl_actual; + } + break; + case BCE_VHCI_PIPE_STALL: + usb_err = USB_ERR_STALLED; + /* + * Mark endpoint stalled so pipe_start will issue + * ENDPOINT_RESET (0x0044) before the next transfer. + */ + tq->stalled = 1; + break; + case BCE_VHCI_ABORT: + usb_err = USB_ERR_CANCELLED; + break; + default: + usb_err = USB_ERR_IOERROR; + break; + } + + tq->active_xfer = NULL; + tq->dma_inflight = 0; + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + + usbd_transfer_done(xfer, usb_err); +} + +/* + * Handle CTRL_TRANSFER_STATUS from firmware. + * + * This signals the end of a control transfer. + * param1 = (ep_addr << 8) | dev_id + * status = BCE_VHCI_SUCCESS(1) or error code + * + * For IN transfers, the IN DMA completion (tq_completion) must have + * set ctrl_actual before we can copy data. If the DMA completion + * has not fired yet (ctrl_data_done == 0), defer this message and + * let tq_completion process it when the data arrives. + */ +static void +bce_vhci_handle_ctrl_status(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg) +{ + struct bce_vhci_transfer_queue *tq; + uint8_t dev_id, ep_addr; + + dev_id = msg->param1 & 0xFF; + ep_addr = (msg->param1 >> 8) & 0xFF; + + tq = bce_vhci_find_tq(vhci, dev_id, ep_addr); + if (tq == NULL) { + device_printf(vhci->sc_dev, + "CTRL_TRANSFER_STATUS for unknown dev=%d ep=0x%02x\n", + dev_id, ep_addr); + return; + } + + /* + * Acquire USB_BUS_LOCK before touching xfer state. + * This is called from ev_generic_completion (ithread context), + * so MTX_DEF is safe. Serializes with pipe_close/pipe_start. + */ + USB_BUS_LOCK(&vhci->sc_bus); + + if (tq->active_xfer == NULL) { + USB_BUS_UNLOCK(&vhci->sc_bus); + device_printf(vhci->sc_dev, + "CTRL_TRANSFER_STATUS but no active xfer\n"); + return; + } + + /* + * Defer successful completion until DMA completes. + * - ctrl_state SETUP: setup packet DMA still in flight + * - ctrl_data_len > 0 with !ctrl_data_done: data DMA pending + * Error statuses are never deferred to avoid permanent hangs + * if the DMA completion is lost due to the error. + */ + if (msg->status == BCE_VHCI_SUCCESS && + (tq->ctrl_state == BCE_VHCI_CTRL_SETUP || + (tq->ctrl_data_len > 0 && tq->ctrl_data_done == 0))) { + tq->ctrl_status_msg = *msg; + tq->ctrl_status_pending = 1; + USB_BUS_UNLOCK(&vhci->sc_bus); + return; + } + + if (msg->status != BCE_VHCI_SUCCESS) + device_printf(vhci->sc_dev, + "CTRL_TRANSFER_STATUS: dev=%d ep=0x%02x status=%u\n", + dev_id, ep_addr, msg->status); + + bce_vhci_complete_ctrl_locked(vhci, tq, msg); + USB_BUS_UNLOCK(&vhci->sc_bus); +} + +/* + * Send a firmware event reply on msg_system. + * + * Firmware events on ev_commands are acknowledged by replying with + * cmd | 0x8000 and a status code on msg_system (NOT msg_asynchronous, + * NOT ENDPOINT_SET_STATE). + */ +static void +bce_vhci_send_fw_event_reply(struct bce_vhci_softc *vhci, + struct bce_vhci_message *req, uint16_t status) +{ + struct bce_vhci_message resp; + + resp.cmd = req->cmd | BCE_VHCI_CMD_REPLY_FLAG; + resp.status = status; + resp.param1 = req->param1; + resp.param2 = 0; + + if (bce_reserve_submission(vhci->msg_system.sq) == 0) + bce_vhci_msg_queue_write(vhci, &vhci->msg_system, &resp); + else + device_printf(vhci->sc_dev, + "failed to send FW event reply for 0x%04x\n", + req->cmd); +} + +/* + * Handle ENDPOINT_REQ_STATE (0x0043) from firmware. + * + * Called from taskqueue context (sole consumer of ev_commands). + * Updates internal pause/stall state only; no messages are sent + * on msg_asynchronous from here (that queue is written from ISR + * context only, avoiding multi-producer races). + * + * The reply (cmd | 0x8000) is sent by the caller on msg_system. + */ +static uint16_t +bce_vhci_handle_endpoint_req_state(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg) +{ + struct bce_vhci_transfer_queue *tq; + uint8_t dev_id, ep_addr; + uint32_t req_state; + + dev_id = msg->param1 & 0xFF; + ep_addr = (msg->param1 >> 8) & 0xFF; + req_state = (uint32_t)msg->param2; + + tq = bce_vhci_find_tq(vhci, dev_id, ep_addr); + if (tq == NULL) + return (BCE_VHCI_BAD_ARGUMENT); + + /* + * USB_BUS_LOCK protects paused_by, stalled, ctrl_state, and active_xfer + * against concurrent access from pipe_start, pipe_close, and ISR paths. + * Nesting USB_BUS_LOCK (MTX_DEF) -> tq->lock / sc_async_lock (MTX_SPIN) + * is valid. + */ + USB_BUS_LOCK(&vhci->sc_bus); + + /* Revalidate after taking the lock; teardown may have started */ + if (tq->active == 0) { + USB_BUS_UNLOCK(&vhci->sc_bus); + return (BCE_VHCI_SUCCESS); + } + + switch (req_state) { + case BCE_VHCI_ENDP_ACTIVE: + { + int was_paused_by_fw; + + was_paused_by_fw = + (tq->paused_by & BCE_VHCI_PAUSE_FIRMWARE) != 0; + tq->paused_by &= ~BCE_VHCI_PAUSE_FIRMWARE; + tq->stalled = 0; + /* + * Firmware flushes SQs during PAUSE, so after ACTIVE we must + * re-submit the IN buffer + TRANSFER_REQUEST, but ONLY if + * the endpoint was actually paused by firmware. Firmware + * also sends ENDP_ACTIVE after a fresh ENDPOINT_CREATE; in + * that case the create_task has already sent the initial + * TRANSFER_REQUEST and a second submission here would confuse + * firmware state. + * + * NOTE: do NOT send ENDPOINT_SET_STATE here; firmware + * already knows the new state (it requested it). The event + * reply from bce_vhci_send_fw_event_reply in fwevt_task is + * the ack. Sending a command from within fwevt_task would + * deadlock because the reply comes back on ev_commands (same + * taskqueue). + */ + if (was_paused_by_fw && + tq->ctrl_state == BCE_VHCI_CTRL_DATA && + tq->ctrl_dir == UE_DIR_IN) { + /* + * Control IN data phase: re-submit on + * msg_asynchronous + */ + struct bce_vhci_message treq; + struct bce_qe_submission *si; + uint32_t dlen; + + dlen = tq->ctrl_data_len; + if (dlen > BCE_VHCI_XFER_BUFSZ) + dlen = BCE_VHCI_XFER_BUFSZ; + + memset(&treq, 0, sizeof(treq)); + treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + treq.param1 = + ((uint32_t)tq->endp_addr << 8) | tq->dev_addr; + treq.param2 = dlen; + + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREREAD); + + /* + * Reserve msg slot first, then SQ -- + * avoids orphaned SQ entry + */ + mtx_lock_spin(&vhci->sc_async_lock); + if (bce_reserve_submission( + vhci->msg_asynchronous.sq) == 0) { + mtx_unlock_spin(&vhci->sc_async_lock); + + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_in) == 0) { + si = bce_next_submission(tq->sq_in); + si->addr = tq->dma_addr; + si->length = dlen; + si->segl_addr = 0; + si->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, + tq->sq_in); + mtx_unlock_spin(&tq->lock); + + mtx_lock_spin(&vhci->sc_async_lock); + bce_vhci_msg_queue_write(vhci, + &vhci->msg_asynchronous, &treq); + mtx_unlock_spin(&vhci->sc_async_lock); + } else { + mtx_unlock_spin(&tq->lock); + mtx_lock_spin(&vhci->sc_async_lock); + atomic_add_int(&vhci-> + msg_asynchronous.sq-> + available_commands, 1); + mtx_unlock_spin(&vhci->sc_async_lock); + device_printf(vhci->sc_dev, + "ctrl resume: SQ full\n"); + if (tq->active_xfer != NULL) { + struct usb_xfer *ax; + ax = tq->active_xfer; + tq->active_xfer = NULL; + tq->dma_inflight = 0; + tq->ctrl_state = + BCE_VHCI_CTRL_IDLE; + usbd_transfer_done(ax, + USB_ERR_IOERROR); + } + } + } else { + mtx_unlock_spin(&vhci->sc_async_lock); + device_printf(vhci->sc_dev, + "ctrl resume: msg full\n"); + if (tq->active_xfer != NULL) { + struct usb_xfer *ax; + ax = tq->active_xfer; + tq->active_xfer = NULL; + tq->dma_inflight = 0; + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + usbd_transfer_done(ax, + USB_ERR_IOERROR); + } + } + } else if (was_paused_by_fw && + tq->endp_addr != 0x00 && (ep_addr & UE_DIR_IN) && + tq->active_xfer != NULL) { + /* + * Interrupt/bulk IN: re-submit after firmware + * PAUSE/ACTIVE + */ + struct bce_vhci_message treq; + struct bce_qe_submission *si; + uint32_t len; + + len = tq->active_xfer->frlengths[0]; + if (len > BCE_VHCI_XFER_BUFSZ) + len = BCE_VHCI_XFER_BUFSZ; + + memset(&treq, 0, sizeof(treq)); + treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + treq.param1 = + ((uint32_t)tq->endp_addr << 8) | tq->dev_addr; + treq.param2 = len; + + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREREAD); + + /* + * Reserve msg slot first, then SQ -- + * avoids orphaned SQ entry + */ + mtx_lock_spin(&vhci->sc_async_lock); + if (bce_reserve_submission( + vhci->msg_asynchronous.sq) == 0) { + mtx_unlock_spin(&vhci->sc_async_lock); + + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_in) == 0) { + si = bce_next_submission(tq->sq_in); + si->addr = tq->dma_addr; + si->length = len; + si->segl_addr = 0; + si->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, + tq->sq_in); + mtx_unlock_spin(&tq->lock); + + mtx_lock_spin(&vhci->sc_async_lock); + bce_vhci_msg_queue_write(vhci, + &vhci->msg_asynchronous, &treq); + mtx_unlock_spin(&vhci->sc_async_lock); + } else { + mtx_unlock_spin(&tq->lock); + mtx_lock_spin(&vhci->sc_async_lock); + atomic_add_int(&vhci-> + msg_asynchronous.sq-> + available_commands, 1); + mtx_unlock_spin(&vhci->sc_async_lock); + device_printf(vhci->sc_dev, + "IN resume: SQ full\n"); + if (tq->active_xfer != NULL) { + struct usb_xfer *ax; + ax = tq->active_xfer; + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(ax, + USB_ERR_IOERROR); + } + } + } else { + mtx_unlock_spin(&vhci->sc_async_lock); + device_printf(vhci->sc_dev, + "IN resume: msg full\n"); + if (tq->active_xfer != NULL) { + struct usb_xfer *ax; + ax = tq->active_xfer; + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(ax, + USB_ERR_IOERROR); + } + } + } + } /* end ENDP_ACTIVE scope */ + break; + case BCE_VHCI_ENDP_PAUSED: + tq->paused_by |= BCE_VHCI_PAUSE_FIRMWARE; + /* + * Do NOT send ENDPOINT_SET_STATE; same deadlock + * reason as ACTIVE above. The event reply is the ack. + */ + /* + * Flush pending SQ submissions after PAUSE. + * Without this, our pre-submitted IN buffer stays in + * firmware's view and confuses the re-submit after RESUME. + * + * bce_cmd_flush_queue sleeps (waits on semaphore), so we + * must drop USB_BUS_LOCK before calling it. QIDs are + * snapshotted under the lock; endpoint_destroy also takes + * the lock before starting teardown, so the qids remain + * valid with firmware while we flush. A double-flush + * (here + endpoint_destroy) is harmless. + */ + if (tq->active) { + struct apple_bce_softc *sc = vhci->sc_bce; + int sq_in_qid = (tq->sq_in != NULL) ? + tq->sq_in->qid : -1; + int sq_out_qid = (tq->sq_out != NULL) ? + tq->sq_out->qid : -1; + + USB_BUS_UNLOCK(&vhci->sc_bus); + if (sq_in_qid >= 0) + bce_cmd_flush_queue(sc->sc_cmd_cmdq, + sc, sq_in_qid); + if (sq_out_qid >= 0) + bce_cmd_flush_queue(sc->sc_cmd_cmdq, + sc, sq_out_qid); + USB_BUS_LOCK(&vhci->sc_bus); + } + break; + default: + USB_BUS_UNLOCK(&vhci->sc_bus); + return (BCE_VHCI_BAD_ARGUMENT); + } + + USB_BUS_UNLOCK(&vhci->sc_bus); + return (BCE_VHCI_SUCCESS); +} + +/* + * Handle unsolicited ENDPOINT_SET_STATE (0x0042) from firmware. + * + * Firmware notifies us of a state change it initiated (e.g., stall + * after a protocol error). + * + * param1 = (ep_addr << 8) | dev_id + * param2 = new_state + */ +static uint16_t +bce_vhci_handle_endpoint_set_state(struct bce_vhci_softc *vhci, + struct bce_vhci_message *msg) +{ + struct bce_vhci_transfer_queue *tq; + uint8_t dev_id, ep_addr; + uint32_t new_state; + + dev_id = msg->param1 & 0xFF; + ep_addr = (msg->param1 >> 8) & 0xFF; + new_state = (uint32_t)msg->param2; + + device_printf(vhci->sc_dev, + "ENDPOINT_SET_STATE: dev=%d ep=0x%02x state=%u\n", + dev_id, ep_addr, new_state); + + tq = bce_vhci_find_tq(vhci, dev_id, ep_addr); + if (tq == NULL) + return (BCE_VHCI_BAD_ARGUMENT); + + switch (new_state) { + case BCE_VHCI_ENDP_STALLED: + /* + * USB_BUS_LOCK protects stalled against concurrent access + * from pipe_start, pipe_close, and the endpoint_req_state path. + */ + USB_BUS_LOCK(&vhci->sc_bus); + tq->stalled = 1; + USB_BUS_UNLOCK(&vhci->sc_bus); + /* + * Do not touch active_xfer here; this runs from + * taskqueue while ISR may be using it. The stall + * will be reported via CTRL_TRANSFER_STATUS(STALL) + * from firmware on the ISR path. + */ + return (BCE_VHCI_SUCCESS); + default: + return (BCE_VHCI_BAD_ARGUMENT); + } +} + +/* + * QID bitmap allocator (internal, caller must hold sc_queues_lock). + * BCE_MAX_QUEUE_COUNT = 256 = 8 * 32 bits. + * Bit set means QID is in use. + */ +static int +bce_vhci_alloc_qid_locked(struct bce_vhci_softc *vhci) +{ + struct apple_bce_softc *sc __unused = vhci->sc_bce; + int i, bit; + + mtx_assert(&sc->sc_queues_lock, MA_OWNED); + for (i = 0; i < 8; i++) { + if (vhci->sc_qid_bitmap[i] == 0xFFFFFFFF) + continue; + bit = ffs(~vhci->sc_qid_bitmap[i]) - 1; + vhci->sc_qid_bitmap[i] |= (1u << bit); + return (i * 32 + bit); + } + return (-1); +} + +static int +bce_vhci_alloc_qid(struct bce_vhci_softc *vhci) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + int qid; + + mtx_lock(&sc->sc_queues_lock); + qid = bce_vhci_alloc_qid_locked(vhci); + mtx_unlock(&sc->sc_queues_lock); + return (qid); +} + +/* + * Allocate next queue ID pair (CQ + SQ). + * Returns the CQ qid; SQ qid = CQ qid + 1. + * Finds two consecutive free bits in the bitmap. + */ +static int +bce_vhci_alloc_qid_pair(struct bce_vhci_softc *vhci) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + int qid, j; + + mtx_lock(&sc->sc_queues_lock); + + /* Find first free QID and check the next one is also free */ + qid = bce_vhci_alloc_qid_locked(vhci); + if (qid < 0) { + mtx_unlock(&sc->sc_queues_lock); + return (-1); + } + if (qid + 1 >= BCE_MAX_QUEUE_COUNT) { + /* Free the one we just allocated */ + vhci->sc_qid_bitmap[qid / 32] &= ~(1u << (qid % 32)); + mtx_unlock(&sc->sc_queues_lock); + return (-1); + } + /* Check next QID is free */ + if (vhci->sc_qid_bitmap[(qid + 1) / 32] & (1u << ((qid + 1) % 32))) { + /* Next is taken; free qid and search for consecutive pair */ + vhci->sc_qid_bitmap[qid / 32] &= ~(1u << (qid % 32)); + /* Brute-force search for consecutive pair */ + for (j = BCE_QUEUE_USER_MIN; j < BCE_MAX_QUEUE_COUNT - 1; j++) { + uint32_t w0 = vhci->sc_qid_bitmap[j / 32]; + uint32_t w1 = vhci->sc_qid_bitmap[(j + 1) / 32]; + int b0 = j % 32; + int b1 = (j + 1) % 32; + + if ((w0 & (1u << b0)) == 0 && + (w1 & (1u << b1)) == 0) { + vhci->sc_qid_bitmap[j / 32] |= (1u << b0); + vhci->sc_qid_bitmap[(j + 1) / 32] |= + (1u << b1); + mtx_unlock(&sc->sc_queues_lock); + return (j); + } + } + mtx_unlock(&sc->sc_queues_lock); + return (-1); + } + /* Next QID is free; allocate it */ + vhci->sc_qid_bitmap[(qid + 1) / 32] |= (1u << ((qid + 1) % 32)); + mtx_unlock(&sc->sc_queues_lock); + return (qid); +} + +static void +bce_vhci_free_qid(struct bce_vhci_softc *vhci, int qid) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + + if (qid < 0 || qid >= BCE_MAX_QUEUE_COUNT) + return; + mtx_lock(&sc->sc_queues_lock); + vhci->sc_qid_bitmap[qid / 32] &= ~(1u << (qid % 32)); + mtx_unlock(&sc->sc_queues_lock); +} + +static int +bce_vhci_create_queues(struct bce_vhci_softc *vhci) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + struct bce_queue_memcfg cfg; + uint32_t status; + int error, qid_pair, q, i; + + /* Initialize QID bitmap: mark QIDs 0..BCE_QUEUE_USER_MIN-1 as used */ + memset(vhci->sc_qid_bitmap, 0, sizeof(vhci->sc_qid_bitmap)); + for (q = 0; q < BCE_QUEUE_USER_MIN; q++) + vhci->sc_qid_bitmap[q / 32] |= (1u << (q % 32)); + + /* Initialize command queue locks early (destroy_queues expects them) */ + sx_init(&vhci->cmd.exec_lock, "bce_vhci_cmdex"); + mtx_init(&vhci->cmd.lock, "bce_vhci_cmd", NULL, MTX_SPIN); + sema_init(&vhci->cmd.completion, 0, "bce_vhci_cmd"); + vhci->cmd.pending = 0; + + /* + * Create 5 message queues (host -> device). + * Each gets its own CQ + SQ pair. + */ +#define CREATE_MSG_QUEUE(field, name) \ + do { \ + qid_pair = bce_vhci_alloc_qid_pair(vhci); \ + if (qid_pair < 0) { \ + error = ENOMEM; \ + device_printf(vhci->sc_dev, \ + "queue IDs exhausted for %s\n", name); \ + goto fail; \ + } \ + error = bce_vhci_msg_queue_create(vhci, &vhci->field, \ + name, qid_pair, qid_pair + 1, \ + bce_vhci_msg_queue_completion, &vhci->field); \ + if (error != 0) { \ + device_printf(vhci->sc_dev, \ + "failed to create %s: %d\n", name, error); \ + goto fail; \ + } \ + } while (0) + + CREATE_MSG_QUEUE(msg_commands, "VHC1HostCommands"); + CREATE_MSG_QUEUE(msg_system, "VHC1HostSystemEvents"); + CREATE_MSG_QUEUE(msg_isochronous, "VHC1HostIsochronousEvents"); + CREATE_MSG_QUEUE(msg_interrupt, "VHC1HostInterruptEvents"); + CREATE_MSG_QUEUE(msg_asynchronous, "VHC1HostAsynchronousEvents"); +#undef CREATE_MSG_QUEUE + + /* + * Create shared event CQ (one CQ for all 5 event queues). + */ + { + int ev_cq_qid = bce_vhci_alloc_qid(vhci); + + if (ev_cq_qid < 0) { + device_printf(vhci->sc_dev, + "queue IDs exhausted for event CQ\n"); + error = ENOMEM; + goto fail; + } + + vhci->ev_cq = bce_alloc_cq(sc, ev_cq_qid, + BCE_VHCI_EVT_QUEUE_EL); + if (vhci->ev_cq == NULL) { + bce_vhci_free_qid(vhci, ev_cq_qid); + error = ENOMEM; + goto fail; + } + + bce_get_cq_memcfg(vhci->ev_cq, &cfg); + cfg.vector_or_cq = 4; + status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, + &cfg, NULL, 0); + if (status != 0) { + device_printf(vhci->sc_dev, + "failed to register event CQ: %u\n", status); + bce_free_cq(sc, vhci->ev_cq); + vhci->ev_cq = NULL; + bce_vhci_free_qid(vhci, ev_cq_qid); + error = EIO; + goto fail; + } + + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[ev_cq_qid] = vhci->ev_cq; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == NULL) { + sc->sc_cq_list[i] = vhci->ev_cq; + break; + } + } + if (i == BCE_MAX_CQ_COUNT) { + sc->sc_queues[ev_cq_qid] = NULL; + mtx_unlock(&sc->sc_queues_lock); + device_printf(vhci->sc_dev, + "CQ list full for event CQ\n"); + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, + ev_cq_qid); + bce_free_cq(sc, vhci->ev_cq); + vhci->ev_cq = NULL; + bce_vhci_free_qid(vhci, ev_cq_qid); + error = ENOSPC; + goto fail; + } + mtx_unlock(&sc->sc_queues_lock); + } + + /* + * Create 5 event queues (device -> host), all sharing ev_cq. + */ +#define CREATE_EVT_QUEUE(field, name, fn) \ + do { \ + int eq_qid = bce_vhci_alloc_qid(vhci); \ + if (eq_qid < 0) { \ + device_printf(vhci->sc_dev, \ + "queue IDs exhausted for %s\n", name); \ + error = ENOMEM; \ + goto fail; \ + } \ + error = bce_vhci_evt_queue_create(vhci, &vhci->field, \ + name, eq_qid, fn); \ + if (error != 0) { \ + device_printf(vhci->sc_dev, \ + "failed to create %s: %d\n", name, error); \ + bce_vhci_free_qid(vhci, eq_qid); \ + goto fail; \ + } \ + } while (0) + + CREATE_EVT_QUEUE(ev_commands, "VHC1FirmwareCommands", + bce_vhci_ev_cmd_completion); + CREATE_EVT_QUEUE(ev_system, "VHC1FirmwareSystemEvents", + bce_vhci_ev_system_completion); + CREATE_EVT_QUEUE(ev_isochronous, "VHC1FirmwareIsochronousEvents", + bce_vhci_ev_generic_completion); + CREATE_EVT_QUEUE(ev_interrupt, "VHC1FirmwareInterruptEvents", + bce_vhci_ev_generic_completion); + CREATE_EVT_QUEUE(ev_asynchronous, "VHC1FirmwareAsynchronousEvents", + bce_vhci_ev_generic_completion); +#undef CREATE_EVT_QUEUE + + /* Wire command queue to its message queue */ + vhci->cmd.msg = &vhci->msg_commands; + + device_printf(vhci->sc_dev, "VHCI queues created\n"); + return (0); + +fail: + bce_vhci_destroy_queues(vhci); + return (error); +} + +static void +bce_vhci_destroy_queues(struct bce_vhci_softc *vhci) +{ + struct apple_bce_softc *sc = vhci->sc_bce; + int i; + + /* Destroy event queues first (they may deliver cmd replies) */ + bce_vhci_evt_queue_destroy(vhci, &vhci->ev_asynchronous); + bce_vhci_evt_queue_destroy(vhci, &vhci->ev_interrupt); + bce_vhci_evt_queue_destroy(vhci, &vhci->ev_isochronous); + bce_vhci_evt_queue_destroy(vhci, &vhci->ev_system); + bce_vhci_evt_queue_destroy(vhci, &vhci->ev_commands); + + /* Destroy command queue state after events are drained */ + sema_destroy(&vhci->cmd.completion); + if (mtx_initialized(&vhci->cmd.lock)) + mtx_destroy(&vhci->cmd.lock); + sx_destroy(&vhci->cmd.exec_lock); + + /* Destroy shared event CQ */ + if (vhci->ev_cq != NULL) { + bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, + vhci->ev_cq->qid); + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[vhci->ev_cq->qid] = NULL; + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] == vhci->ev_cq) { + sc->sc_cq_list[i] = NULL; + break; + } + } + mtx_unlock(&sc->sc_queues_lock); + bce_free_cq(sc, vhci->ev_cq); + vhci->ev_cq = NULL; + } + + /* Destroy message queues */ + bce_vhci_msg_queue_destroy(vhci, &vhci->msg_asynchronous); + bce_vhci_msg_queue_destroy(vhci, &vhci->msg_interrupt); + bce_vhci_msg_queue_destroy(vhci, &vhci->msg_isochronous); + bce_vhci_msg_queue_destroy(vhci, &vhci->msg_system); + bce_vhci_msg_queue_destroy(vhci, &vhci->msg_commands); +} + +static int +bce_vhci_start_controller(struct bce_vhci_softc *vhci) +{ + struct bce_vhci_message cmd, reply; + uint16_t port_mask; + uint8_t port_count; + uint32_t port_status; + int error; + int i; + + /* + * CONTROLLER_ENABLE: param1 = 0x7100 | bus_number(1) + * Reply param2 = port bitmask + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_ENABLE; + cmd.param1 = 0x7100 | 1; + + error = bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_LONG); + if (error != 0) { + device_printf(vhci->sc_dev, + "CONTROLLER_ENABLE failed: %d\n", error); + return (error); + } + + port_mask = (uint16_t)reply.param2; + vhci->sc_port_mask = port_mask; + + /* Count ports from mask */ + port_count = 0; + for (i = 0; i < BCE_VHCI_MAX_PORTS; i++) { + if (port_mask & (1u << i)) + port_count = i + 1; + } + vhci->sc_port_count = port_count; + + device_printf(vhci->sc_dev, + "controller enabled: port_mask=0x%x, %d ports\n", + port_mask, port_count); + + /* + * CONTROLLER_START + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_START; + + error = bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_LONG); + if (error != 0) { + device_printf(vhci->sc_dev, + "CONTROLLER_START failed: %d\n", error); + /* Disable the controller we just enabled */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE; + bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_LONG); + return (error); + } + + vhci->sc_started = 1; + + /* + * Power on each port and read initial status. + */ + for (i = 0; i < port_count; i++) { + if ((port_mask & (1u << i)) == 0) + continue; + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_PORT_POWER_ON; + cmd.param1 = i; + + error = bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_SHORT); + if (error != 0) { + device_printf(vhci->sc_dev, + "PORT_POWER_ON(%d) failed: %d\n", i, error); + continue; + } + vhci->sc_port_power[i] = 1; + + /* Read initial port status */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_PORT_STATUS; + cmd.param1 = i; + + error = bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_SHORT); + if (error != 0) { + device_printf(vhci->sc_dev, + "PORT_STATUS(%d) failed: %d\n", i, error); + continue; + } + + port_status = (uint32_t)reply.param2; + device_printf(vhci->sc_dev, + "port %d: raw_status=0x%x\n", i, port_status); + + /* + * Translate firmware port status to USB status bits. + */ + vhci->sc_port_status[i] = UPS_PORT_POWER; + if (port_status & BCE_VHCI_PORT_ENABLED) + vhci->sc_port_status[i] |= + UPS_PORT_ENABLED | UPS_HIGH_SPEED; + if (port_status & BCE_VHCI_PORT_CONNECTED) + vhci->sc_port_status[i] |= + UPS_CURRENT_CONNECT_STATUS; + if (port_status & BCE_VHCI_PORT_SUSPENDED) + vhci->sc_port_status[i] |= UPS_SUSPEND; + if (port_status & BCE_VHCI_PORT_OVERCURRENT) + vhci->sc_port_status[i] |= + UPS_OVERCURRENT_INDICATOR; + + if (vhci->sc_port_status[i] & UPS_CURRENT_CONNECT_STATUS) + vhci->sc_port_change[i] |= UPS_C_CONNECT_STATUS; + } + + return (0); +} + +static usb_error_t +bce_vhci_roothub_exec(struct usb_device *udev, + struct usb_device_request *req, const void **pptr, uint16_t *plength) +{ + struct bce_vhci_softc *vhci; + const void *ptr; + uint16_t len; + uint16_t value, index; + usb_error_t err; + + vhci = (struct bce_vhci_softc *)udev->bus; + USB_BUS_LOCK_ASSERT(&vhci->sc_bus, MA_OWNED); + + ptr = NULL; + len = 0; + err = 0; + + value = UGETW(req->wValue); + index = UGETW(req->wIndex); + + switch (req->bRequest) { + case UR_CLEAR_FEATURE: + switch (req->bmRequestType) { + case UT_WRITE_CLASS_OTHER: + /* ClearPortFeature */ + if (index < 1 || index > vhci->sc_port_count) { + err = USB_ERR_IOERROR; + break; + } + switch (value) { + case UHF_C_PORT_CONNECTION: + vhci->sc_port_change[index - 1] &= + ~UPS_C_CONNECT_STATUS; + break; + case UHF_C_PORT_ENABLE: + vhci->sc_port_change[index - 1] &= + ~UPS_C_PORT_ENABLED; + break; + case UHF_C_PORT_RESET: + vhci->sc_port_change[index - 1] &= + ~UPS_C_PORT_RESET; + break; + case UHF_C_PORT_OVER_CURRENT: + vhci->sc_port_change[index - 1] &= + ~UPS_C_OVERCURRENT_INDICATOR; + break; + case UHF_C_PORT_SUSPEND: + vhci->sc_port_change[index - 1] &= + ~UPS_C_SUSPEND; + break; + case UHF_PORT_ENABLE: + vhci->sc_port_status[index - 1] &= + ~UPS_PORT_ENABLED; + break; + case UHF_PORT_SUSPEND: + vhci->sc_port_status[index - 1] &= + ~UPS_SUSPEND; + break; + case UHF_PORT_POWER: + if (vhci->sc_port_power[index - 1]) { + struct bce_vhci_message cmd_pw; + struct bce_vhci_message reply_pw; + int pw_port = index - 1; + + memset(&cmd_pw, 0, sizeof(cmd_pw)); + cmd_pw.cmd = + BCE_VHCI_CMD_PORT_POWER_OFF; + cmd_pw.param1 = pw_port; + + USB_BUS_UNLOCK(&vhci->sc_bus); + if (bce_vhci_cmd_execute(vhci, &cmd_pw, + &reply_pw, + BCE_VHCI_CMD_TIMEOUT_SHORT) != 0) { + device_printf(vhci->sc_dev, + "PORT_POWER_OFF(%d)" + " failed\n", pw_port); + USB_BUS_LOCK(&vhci->sc_bus); + err = USB_ERR_IOERROR; + break; + } + USB_BUS_LOCK(&vhci->sc_bus); + vhci->sc_port_power[pw_port] = 0; + vhci->sc_port_status[pw_port] = 0; + } + break; + default: + err = USB_ERR_IOERROR; + break; + } + break; + default: + err = USB_ERR_IOERROR; + break; + } + break; + + case UR_GET_DESCRIPTOR: + if (req->bmRequestType == UT_READ_CLASS_DEVICE) { + /* Hub descriptor (USB 2.0) */ + struct usb_hub_descriptor hd; + uint8_t nports = vhci->sc_port_count; + uint8_t padsz = (nports + 7) / 8; + + memset(&hd, 0, sizeof(hd)); + hd.bDescLength = 7 + 2 * padsz; + hd.bDescriptorType = UDESC_HUB; + hd.bNbrPorts = nports; + USETW(hd.wHubCharacteristics, + UHD_PWR_INDIVIDUAL); + hd.bPwrOn2PwrGood = 50; + len = hd.bDescLength; + if (len > sizeof(vhci->sc_hub_idata)) + len = sizeof(vhci->sc_hub_idata); + memcpy(vhci->sc_hub_idata, &hd, len); + ptr = vhci->sc_hub_idata; + break; + } + switch (value >> 8) { + case UDESC_DEVICE: + if ((value & 0xff) != 0) { + err = USB_ERR_IOERROR; + break; + } + len = sizeof(bce_vhci_devd); + ptr = &bce_vhci_devd; + break; + case UDESC_DEVICE_QUALIFIER: + if ((value & 0xff) != 0) { + err = USB_ERR_IOERROR; + break; + } + len = sizeof(bce_vhci_odevd); + ptr = &bce_vhci_odevd; + break; + case UDESC_CONFIG: + if ((value & 0xff) != 0) { + err = USB_ERR_IOERROR; + break; + } + len = sizeof(bce_vhci_confd); + ptr = bce_vhci_confd; + break; + case UDESC_STRING: + switch (value & 0xff) { + case 0: /* Language */ + ptr = "\x04\x03\x09\x04"; + len = 4; + break; + case 1: /* Vendor */ + ptr = "\x0c\x03\x41\x00\x70\x00\x70\x00" + "\x6c\x00\x65\x00"; + len = 12; + break; + case 2: /* Product */ + ptr = "\x1a\x03\x54\x00\x32\x00\x20\x00" + "\x42\x00\x43\x00\x45\x00\x20\x00" + "\x56\x00\x48\x00\x43\x00\x49\x00"; + len = 26; + break; + default: + err = USB_ERR_IOERROR; + break; + } + break; + default: + err = USB_ERR_IOERROR; + break; + } + break; + + case UR_GET_INTERFACE: + len = 1; + ptr = "\x00"; /* alt setting 0 */ + break; + + case UR_GET_STATUS: + switch (req->bmRequestType) { + case UT_READ_CLASS_OTHER: + { + /* GetPortStatus */ + struct usb_port_status ps; + uint16_t port; + + if (index < 1 || index > vhci->sc_port_count) { + err = USB_ERR_IOERROR; + break; + } + port = index - 1; + + USETW(ps.wPortStatus, + vhci->sc_port_status[port]); + USETW(ps.wPortChange, + vhci->sc_port_change[port]); + + len = sizeof(ps); + memcpy(&vhci->sc_hub_idata, &ps, sizeof(ps)); + ptr = &vhci->sc_hub_idata; + break; + } + case UT_READ_CLASS_DEVICE: + { + /* GetHubStatus */ + len = 4; + ptr = "\x00\x00\x00\x00"; + break; + } + case UT_READ_DEVICE: + { + len = 2; + ptr = "\x01\x00"; /* self-powered */ + break; + } + default: + err = USB_ERR_IOERROR; + break; + } + break; + + case UR_SET_ADDRESS: + if (value >= BCE_VHCI_MAX_DEVICES) { + err = USB_ERR_IOERROR; + break; + } + break; + + case UR_SET_CONFIG: + case UR_SET_INTERFACE: + break; + + case UR_SET_FEATURE: + switch (req->bmRequestType) { + case UT_WRITE_CLASS_OTHER: + /* SetPortFeature */ + if (index < 1 || index > vhci->sc_port_count) { + err = USB_ERR_IOERROR; + break; + } + switch (value) { + case UHF_PORT_POWER: + if (vhci->sc_port_power[index - 1] == 0) { + struct bce_vhci_message cmd_pw; + struct bce_vhci_message reply_pw; + int pw_port = index - 1; + int pw_err; + + memset(&cmd_pw, 0, sizeof(cmd_pw)); + cmd_pw.cmd = BCE_VHCI_CMD_PORT_POWER_ON; + cmd_pw.param1 = pw_port; + + USB_BUS_UNLOCK(&vhci->sc_bus); + pw_err = bce_vhci_cmd_execute(vhci, + &cmd_pw, &reply_pw, + BCE_VHCI_CMD_TIMEOUT_SHORT); + USB_BUS_LOCK(&vhci->sc_bus); + if (pw_err != 0) { + device_printf(vhci->sc_dev, + "PORT_POWER_ON(%d)" + " failed\n", pw_port); + err = USB_ERR_IOERROR; + } else { + vhci->sc_port_power[pw_port] = + 1; + vhci->sc_port_status[pw_port] |= + UPS_PORT_POWER; + } + } + break; + case UHF_PORT_RESET: + { + int reset_err; + + vhci->sc_port_status[index - 1] |= + UPS_RESET; + + /* + * Drop bus lock for firmware I/O. + * Explore thread is single-threaded + * so this is safe. + */ + USB_BUS_UNLOCK(&vhci->sc_bus); + + /* Destroy existing device before re-creating */ + bce_vhci_device_destroy(vhci, index - 1); + + reset_err = bce_vhci_device_create(vhci, + index - 1); + USB_BUS_LOCK(&vhci->sc_bus); + + vhci->sc_port_status[index - 1] &= + ~UPS_RESET; + if (reset_err == 0) { + vhci->sc_port_status[index - 1] |= + UPS_PORT_ENABLED | + UPS_HIGH_SPEED; + vhci->sc_port_change[index - 1] |= + UPS_C_PORT_RESET; + } else { + device_printf(vhci->sc_dev, + "port %d reset failed: %d\n", + index, reset_err); + err = USB_ERR_IOERROR; + } + break; + } + case UHF_PORT_ENABLE: + vhci->sc_port_status[index - 1] |= + UPS_PORT_ENABLED; + break; + case UHF_PORT_SUSPEND: + vhci->sc_port_status[index - 1] |= + UPS_SUSPEND; + break; + default: + err = USB_ERR_IOERROR; + break; + } + break; + default: + err = USB_ERR_IOERROR; + break; + } + break; + + case UR_GET_CONFIG: + len = 1; + ptr = "\x01"; /* config 1 */ + break; + + default: + err = USB_ERR_IOERROR; + break; + } + + if (err == 0) { + if (pptr != NULL) + *pptr = ptr; + if (plength != NULL) + *plength = len; + } + return (err); +} + +static void +bce_vhci_endpoint_init(struct usb_device *udev, + struct usb_endpoint_descriptor *edesc, struct usb_endpoint *ep) +{ + + ep->methods = &bce_vhci_pipe_methods; +} + +static void +bce_vhci_xfer_setup(struct usb_setup_params *parm) +{ + struct usb_xfer *xfer = parm->curr_xfer; + + parm->hc_max_packet_size = 0x400; /* 1024 */ + parm->hc_max_packet_count = 1; + parm->hc_max_frame_size = BCE_VHCI_XFER_BUFSZ; + + usbd_transfer_setup_sub(parm); + + if (parm->err) + return; + + /* No HCD-specific TD/QH structures needed */ + xfer->flags_int.bdma_enable = 0; +} + +static void +bce_vhci_xfer_unsetup(struct usb_xfer *xfer) +{ + /* Nothing to free */ +} + +static void +bce_vhci_get_dma_delay(struct usb_device *udev, uint32_t *pus) +{ + + *pus = 0; /* No hardware DMA delay */ +} + +static void +bce_vhci_pipe_open(struct usb_xfer *xfer) +{ + /* Nothing to do; endpoint resources managed elsewhere */ +} + +static void +bce_vhci_pipe_close(struct usb_xfer *xfer) +{ + struct bce_vhci_softc *vhci; + int i; + + vhci = (struct bce_vhci_softc *)xfer->xroot->bus; + + /* + * If this xfer is the active transfer on any endpoint, + * clear it. We do not flush the firmware SQ here because + * USB_BUS_LOCK is held (cannot sleep). Stale SQ completions + * are discarded in tq_completion (active_xfer == NULL check). + */ + for (i = 0; i < BCE_VHCI_MAX_DEVICES; i++) { + struct bce_vhci_device *dev = &vhci->sc_devs[i]; + int j; + + if (dev->allocated == 0) + continue; + for (j = 0; j < BCE_VHCI_MAX_ENDPOINTS; j++) { + struct bce_vhci_transfer_queue *tq = &dev->tq[j]; + + if (tq->active_xfer == xfer) { + tq->active_xfer = NULL; + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + tq->ctrl_data_done = 0; + tq->ctrl_status_pending = 0; + tq->evt_pending = 0; + } + if (tq->pending_xfer == xfer) + tq->pending_xfer = NULL; + if (tq->create_xfer == xfer) { + tq->create_xfer = NULL; + tq->create_pending = 0; + } + } + } + + /* Cancel any pending transfer */ + if (xfer->flags_int.transferring) { + usbd_transfer_done(xfer, USB_ERR_CANCELLED); + } +} + +static void +bce_vhci_pipe_enter(struct usb_xfer *xfer) +{ + /* Called before start, can validate */ +} + +/* + * Start a transfer. + * + * Called with USB_BUS_LOCK held. For control transfers, we parse the + * setup packet, record the direction and data length, set the + * endpoint state machine to SETUP, and wait for firmware + * TRANSFER_REQUEST events to drive the transfer forward. + * + * For interrupt/bulk, we STALL for now (not yet implemented). + */ +static void +bce_vhci_pipe_start(struct usb_xfer *xfer) +{ + struct bce_vhci_softc *vhci; + struct bce_vhci_device *dev; + struct bce_vhci_transfer_queue *tq; + struct usb_device_request setup; + uint8_t xfer_type; + uint8_t ep_addr; + + vhci = (struct bce_vhci_softc *)xfer->xroot->bus; + xfer_type = xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE; + ep_addr = xfer->endpointno; + + if (xfer_type == UE_INTERRUPT || xfer_type == UE_BULK) { + struct usb_device *udev = xfer->xroot->udev; + struct bce_vhci_message treq; + uint8_t port, fw_dev_id, ep_idx; + uint32_t len; + + port = udev->port_no; + if (port < 1 || port > vhci->sc_port_count) { + usbd_transfer_done(xfer, USB_ERR_STALLED); + return; + } + + fw_dev_id = vhci->sc_port_to_dev[port - 1]; + if (fw_dev_id >= BCE_VHCI_MAX_DEVICES) { + usbd_transfer_done(xfer, USB_ERR_STALLED); + return; + } + + dev = &vhci->sc_devs[fw_dev_id]; + if (dev->allocated == 0) { + usbd_transfer_done(xfer, USB_ERR_STALLED); + return; + } + + ep_idx = bce_vhci_ep_index(ep_addr); + tq = &dev->tq[ep_idx]; + + /* Create endpoint with firmware if not yet active */ + if (tq->active == 0) { + /* + * Cannot sleep here: we may be in a USB callback + * (e.g. usbhid_intr_in_callback) holding a + * non-sleepable lock. Defer to taskqueue_thread + * and return STALLED so the USB stack retries. + */ + if (vhci->sc_detaching == 0 && + tq->create_xfer == NULL) { + tq->endp_addr = ep_addr; + tq->dev_addr = fw_dev_id; + tq->create_pending = 1; + tq->create_edesc = xfer->endpoint->edesc; + /* held; task submits it */ + tq->create_xfer = xfer; + taskqueue_enqueue(taskqueue_thread, + &vhci->sc_create_task); + } else { + usbd_transfer_done(xfer, USB_ERR_STALLED); + } + return; + } + + if (tq->active_xfer != NULL || tq->dma_inflight != 0) { + /* + * Pipeline: queue xfer for when active_xfer finishes. + * Also queue if old DMA is still inflight (pipe_close + * cleared active_xfer but completion not yet seen). + * Returning STALLED triggers stall recovery + * (CLEAR_FEATURE loop); hold the xfer instead. + */ + if (tq->pending_xfer == NULL) + tq->pending_xfer = xfer; + else + usbd_transfer_done(xfer, USB_ERR_CANCELLED); + return; + } + + tq->active_xfer = xfer; + tq->dma_inflight = 1; + + /* + * If firmware already sent a TRANSFER_REQUEST before + * the USB stack called pipe_start, replay it now + * instead of sending a duplicate host request. + */ + if (tq->evt_pending) { + tq->evt_pending = 0; + bce_vhci_handle_transfer_request(vhci, + &tq->evt_saved); + return; + } + + if (ep_addr & UE_DIR_IN) { + /* IN transfer: reserve msg first, then SQ */ + struct bce_qe_submission *si; + + len = xfer->frlengths[0]; + if (len > BCE_VHCI_XFER_BUFSZ) + len = BCE_VHCI_XFER_BUFSZ; + + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREREAD); + + memset(&treq, 0, sizeof(treq)); + treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + treq.param1 = (ep_addr << 8) | tq->dev_addr; + treq.param2 = len; + + /* Reserve msg slot first */ + mtx_lock_spin(&vhci->sc_async_lock); + if (bce_reserve_submission( + vhci->msg_asynchronous.sq) != 0) { + mtx_unlock_spin(&vhci->sc_async_lock); + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, USB_ERR_IOERROR); + return; + } + mtx_unlock_spin(&vhci->sc_async_lock); + + /* Then reserve SQ slot */ + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_in) != 0) { + mtx_unlock_spin(&tq->lock); + /* Restore msg slot reserved but won't use */ + mtx_lock_spin(&vhci->sc_async_lock); + atomic_add_int(&vhci-> + msg_asynchronous.sq-> + available_commands, 1); + mtx_unlock_spin(&vhci->sc_async_lock); + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, USB_ERR_IOERROR); + return; + } + + si = bce_next_submission(tq->sq_in); + si->addr = tq->dma_addr; + si->length = len; + si->segl_addr = 0; + si->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, tq->sq_in); + mtx_unlock_spin(&tq->lock); + + mtx_lock_spin(&vhci->sc_async_lock); + bce_vhci_msg_queue_write(vhci, + &vhci->msg_asynchronous, &treq); + mtx_unlock_spin(&vhci->sc_async_lock); + } else { + /* OUT transfer: reserve msg first, then SQ */ + struct bce_qe_submission *so; + + len = xfer->frlengths[0]; + if (len > BCE_VHCI_XFER_BUFSZ) + len = BCE_VHCI_XFER_BUFSZ; + + usbd_copy_out(&xfer->frbuffers[0], 0, + tq->dma_buf, len); + bus_dmamap_sync(tq->dma_tag, tq->dma_map, + BUS_DMASYNC_PREWRITE); + + memset(&treq, 0, sizeof(treq)); + treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + treq.param1 = (ep_addr << 8) | tq->dev_addr; + treq.param2 = len; + + /* Reserve msg slot first */ + mtx_lock_spin(&vhci->sc_async_lock); + if (bce_reserve_submission( + vhci->msg_asynchronous.sq) != 0) { + mtx_unlock_spin(&vhci->sc_async_lock); + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, USB_ERR_IOERROR); + return; + } + mtx_unlock_spin(&vhci->sc_async_lock); + + /* Then reserve SQ slot */ + mtx_lock_spin(&tq->lock); + if (bce_reserve_submission(tq->sq_out) == 0) { + so = bce_next_submission(tq->sq_out); + so->addr = tq->dma_addr; + so->length = len; + so->segl_addr = 0; + so->segl_length = 0; + bce_submit_to_device(vhci->sc_bce, + tq->sq_out); + } else { + mtx_unlock_spin(&tq->lock); + /* Restore msg slot reserved but won't use */ + mtx_lock_spin(&vhci->sc_async_lock); + atomic_add_int(&vhci-> + msg_asynchronous.sq-> + available_commands, 1); + mtx_unlock_spin(&vhci->sc_async_lock); + tq->active_xfer = NULL; + tq->dma_inflight = 0; + usbd_transfer_done(xfer, USB_ERR_IOERROR); + return; + } + mtx_unlock_spin(&tq->lock); + + mtx_lock_spin(&vhci->sc_async_lock); + bce_vhci_msg_queue_write(vhci, + &vhci->msg_asynchronous, &treq); + mtx_unlock_spin(&vhci->sc_async_lock); + } + return; + } + + if (xfer_type != UE_CONTROL) { + device_printf(vhci->sc_dev, + "xfer start ep=0x%02x type=%d (not supported)\n", + ep_addr, xfer_type); + usbd_transfer_done(xfer, USB_ERR_STALLED); + return; + } + + /* + * Control transfer on ep0. Map the USB device's port number + * to the firmware device ID. The USB stack's port_no comes + * from our root hub, so it maps directly to our port index. + * + * For the root hub itself, the USB stack handles it via + * roothub_exec, so we should never see it here. + */ + { + struct usb_device *udev = xfer->xroot->udev; + uint8_t port, fw_dev_id; + + port = udev->port_no; + if (port < 1 || port > vhci->sc_port_count) { + device_printf(vhci->sc_dev, + "control xfer: invalid port %d\n", port); + usbd_transfer_done(xfer, USB_ERR_STALLED); + return; + } + + fw_dev_id = vhci->sc_port_to_dev[port - 1]; + if (fw_dev_id >= BCE_VHCI_MAX_DEVICES) { + device_printf(vhci->sc_dev, + "control xfer: no firmware device for " + "port %d\n", port); + usbd_transfer_done(xfer, USB_ERR_STALLED); + return; + } + + dev = &vhci->sc_devs[fw_dev_id]; + if (dev->allocated == 0 || dev->tq[0].active == 0) { + device_printf(vhci->sc_dev, + "control xfer: device %d ep0 not ready\n", + fw_dev_id); + usbd_transfer_done(xfer, USB_ERR_STALLED); + return; + } + + tq = &dev->tq[0]; + } + + /* + * If the endpoint is stalled from a previous transfer, we need + * ENDPOINT_RESET (0x0044) before the next transfer can succeed. + * We cannot sleep in pipe_start (USB device mutex held), so + * the reset runs asynchronously on taskqueue_thread. + * + * Return USB_ERR_STALLED to the USB stack so it retries; the + * retry will succeed once sc_reset_task clears tq->stalled. + */ + if (tq->stalled) { + device_printf(vhci->sc_dev, + "control xfer: ep0 stalled, scheduling ENDPOINT_RESET " + "(dev=%d)\n", tq->dev_addr); + if (vhci->sc_detaching == 0) + taskqueue_enqueue(taskqueue_thread, + &vhci->sc_reset_task); + usbd_transfer_done(xfer, USB_ERR_STALLED); + return; + } + + if (tq->active_xfer != NULL || tq->dma_inflight != 0) { + device_printf(vhci->sc_dev, + "control xfer: ep0 busy (dev=%d)\n", tq->dev_addr); + usbd_transfer_done(xfer, USB_ERR_STALLED); + return; + } + + /* Read the 8-byte setup packet from frbuffers[0] */ + usbd_copy_out(&xfer->frbuffers[0], 0, &setup, sizeof(setup)); + + /* Determine data direction and length from setup packet */ + tq->ctrl_dir = (setup.bmRequestType & UT_READ) ? + UE_DIR_IN : UE_DIR_OUT; + tq->ctrl_data_len = UGETW(setup.wLength); + tq->ctrl_actual = 0; + tq->ctrl_data_done = 0; + tq->ctrl_status_pending = 0; + tq->active_xfer = xfer; + tq->ctrl_state = BCE_VHCI_CTRL_SETUP; + + + /* + * SET_ADDRESS is handled by firmware via DEVICE_CREATE -- + * complete immediately without forwarding to firmware. + */ + if (setup.bRequest == UR_SET_ADDRESS) { + tq->ctrl_state = BCE_VHCI_CTRL_IDLE; + tq->active_xfer = NULL; + xfer->aframes = xfer->nframes; + usbd_transfer_done(xfer, USB_ERR_NORMAL_COMPLETION); + return; + } + + /* + * Check for a deferred TRANSFER_REQUEST that arrived before + * pipe_start. If one is pending, replay it now. + */ + if (tq->evt_pending) { + tq->evt_pending = 0; + bce_vhci_handle_transfer_request(vhci, &tq->evt_saved); + } +} + +/* + * DMA tag callback (required by usb_bus_mem_alloc_all) + */ + +static void +bce_vhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb) +{ + /* No hardware-specific DMA pages needed */ +} + +static int +bce_vhci_probe(device_t dev) +{ + + device_set_desc(dev, "Apple T2 BCE Virtual USB Host Controller"); + return (BUS_PROBE_DEFAULT); +} + +static int +bce_vhci_attach_dev(device_t dev) +{ + struct bce_vhci_softc *vhci; + struct apple_bce_softc *bce; + int err; + + vhci = device_get_softc(dev); + bce = device_get_softc(device_get_parent(dev)); + if (bce == NULL) { + device_printf(dev, "no BCE parent\n"); + return (ENXIO); + } + + vhci->sc_dev = dev; + vhci->sc_bce = bce; + + /* Sanity check parent state */ + if (bce->sc_cmd_cmdq == NULL || bce->sc_dma_tag == NULL) { + device_printf(dev, + "BCE parent not ready (cmdq=%p dma_tag=%p)\n", + bce->sc_cmd_cmdq, bce->sc_dma_tag); + return (ENXIO); + } + + mtx_init(&vhci->sc_async_lock, "bce_vhci_async", NULL, MTX_SPIN); + mtx_init(&vhci->sc_fwevt_lock, "bce_vhci_fwevt", NULL, MTX_SPIN); + + /* Initialize device state */ + memset(vhci->sc_devs, 0, sizeof(vhci->sc_devs)); + memset(vhci->sc_port_to_dev, 0xFF, sizeof(vhci->sc_port_to_dev)); + + TASK_INIT(&vhci->sc_fwevt_task, 0, bce_vhci_fwevt_task, vhci); + TASK_INIT(&vhci->sc_reset_task, 0, bce_vhci_reset_task, vhci); + TASK_INIT(&vhci->sc_create_task, 0, bce_vhci_create_task, vhci); + TASK_INIT(&vhci->sc_port_chg_task, 0, bce_vhci_port_chg_task, vhci); + + /* + * Initialize USB bus early so bus_mtx is valid before + * firmware events can call USB_BUS_LOCK. + */ + vhci->sc_bus.parent = dev; + vhci->sc_bus.devices = vhci->sc_devices; + vhci->sc_bus.devices_max = BCE_VHCI_MAX_DEVICES; + vhci->sc_bus.dma_bits = 32; + vhci->sc_bus.usbrev = USB_REV_2_0; + vhci->sc_bus.methods = &bce_vhci_bus_methods; + + err = usb_bus_mem_alloc_all(&vhci->sc_bus, + USB_GET_DMA_TAG(dev), &bce_vhci_iterate_hw_softc); + if (err != 0) { + device_printf(dev, "usb_bus_mem_alloc_all failed: %d\n", err); + goto fail; + } + + /* + * Create BCE message/event queues for VHCI communication. + */ + err = bce_vhci_create_queues(vhci); + if (err != 0) { + device_printf(dev, "failed to create VHCI queues: %d\n", err); + goto fail_mem; + } + + /* + * Start controller: ENABLE -> discover ports -> START -> power on. + */ + err = bce_vhci_start_controller(vhci); + if (err != 0) { + device_printf(dev, + "failed to start VHCI controller: %d\n", err); + goto fail_queues; + } + + /* Create usbus child */ + vhci->sc_bus.bdev = device_add_child(dev, "usbus", DEVICE_UNIT_ANY); + if (vhci->sc_bus.bdev == NULL) { + device_printf(dev, "failed to add usbus child\n"); + err = ENOMEM; + goto fail_ctrl; + } + device_set_ivars(vhci->sc_bus.bdev, &vhci->sc_bus); + + err = device_probe_and_attach(vhci->sc_bus.bdev); + if (err != 0) { + device_printf(dev, "usbus attach failed: %d\n", err); + goto fail_child; + } + + device_printf(dev, "BCE VHCI attached, %d ports\n", + vhci->sc_port_count); + return (0); + +fail_child: + device_delete_child(dev, vhci->sc_bus.bdev); +fail_ctrl: + /* Disable controller before tearing down queues */ + if (vhci->sc_started && vhci->msg_commands.sq != NULL) { + struct bce_vhci_message cmd_dis, reply_dis; + + memset(&cmd_dis, 0, sizeof(cmd_dis)); + cmd_dis.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE; + bce_vhci_cmd_execute(vhci, &cmd_dis, &reply_dis, + BCE_VHCI_CMD_TIMEOUT_LONG); + vhci->sc_started = 0; + } +fail_queues: + vhci->sc_detaching = 1; + taskqueue_drain(taskqueue_thread, &vhci->sc_reset_task); + taskqueue_drain(taskqueue_thread, &vhci->sc_create_task); + taskqueue_drain(taskqueue_thread, &vhci->sc_port_chg_task); + taskqueue_drain(taskqueue_thread, &vhci->sc_fwevt_task); + bce_vhci_destroy_queues(vhci); +fail_mem: + usb_bus_mem_free_all(&vhci->sc_bus, &bce_vhci_iterate_hw_softc); +fail: + mtx_destroy(&vhci->sc_fwevt_lock); + mtx_destroy(&vhci->sc_async_lock); + return (err); +} + +static int +bce_vhci_detach_dev(device_t dev) +{ + struct bce_vhci_softc *vhci; + + vhci = device_get_softc(dev); + + /* Stop deferred work before tearing down USB child */ + vhci->sc_detaching = 1; + + /* Detach usbus child */ + if (vhci->sc_bus.bdev != NULL) { + int err; + + err = device_delete_children(dev); + if (err != 0) { + vhci->sc_detaching = 0; + return (err); + } + vhci->sc_bus.bdev = NULL; + } + + /* + * Drain tasks that may reference tq state before + * destroying endpoints. + */ + taskqueue_drain(taskqueue_thread, &vhci->sc_reset_task); + taskqueue_drain(taskqueue_thread, &vhci->sc_create_task); + taskqueue_drain(taskqueue_thread, &vhci->sc_port_chg_task); + + /* Destroy all firmware devices and their endpoints */ + { + int i; + + for (i = 0; i < BCE_VHCI_MAX_PORTS; i++) + bce_vhci_device_destroy(vhci, i); + } + + /* Send CONTROLLER_DISABLE if we started */ + if (vhci->sc_started && vhci->msg_commands.sq != NULL) { + struct bce_vhci_message cmd, reply; + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE; + bce_vhci_cmd_execute(vhci, &cmd, &reply, + BCE_VHCI_CMD_TIMEOUT_LONG); + vhci->sc_started = 0; + } + + /* Drain firmware event task after command completion */ + taskqueue_drain(taskqueue_thread, &vhci->sc_fwevt_task); + + /* Tear down VHCI queues (unregisters from IRQ dispatch first) */ + bce_vhci_destroy_queues(vhci); + + usb_bus_mem_free_all(&vhci->sc_bus, &bce_vhci_iterate_hw_softc); + mtx_destroy(&vhci->sc_fwevt_lock); + mtx_destroy(&vhci->sc_async_lock); + + return (0); +} + +int +bce_vhci_attach(struct apple_bce_softc *sc) +{ + device_t vhci_dev; + + vhci_dev = device_add_child(sc->sc_dev, "bce_vhci", DEVICE_UNIT_ANY); + if (vhci_dev == NULL) { + device_printf(sc->sc_dev, + "failed to add bce_vhci child\n"); + return (ENOMEM); + } + sc->sc_vhci_dev = vhci_dev; + device_set_ivars(vhci_dev, sc); + + if (device_probe_and_attach(vhci_dev) != 0) { + device_delete_child(sc->sc_dev, vhci_dev); + sc->sc_vhci_dev = NULL; + return (ENXIO); + } + return (0); +} + +int +bce_vhci_detach(struct apple_bce_softc *sc) +{ + device_t vhci_dev; + int error; + + vhci_dev = sc->sc_vhci_dev; + if (vhci_dev == NULL) + return (0); + + error = device_delete_child(sc->sc_dev, vhci_dev); + if (error == 0) + sc->sc_vhci_dev = NULL; + + return (error); +} diff --git a/sys/dev/apple_bce/apple_bce_vhci.h b/sys/dev/apple_bce/apple_bce_vhci.h new file mode 100644 index 00000000000..c85dd326b27 --- /dev/null +++ b/sys/dev/apple_bce/apple_bce_vhci.h @@ -0,0 +1,251 @@ +/*- + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Apple T2 BCE Virtual USB Host Controller Interface (VHCI). + * Translates USB operations into BCE firmware messages over DMA queues. + */ + +#ifndef _APPLE_BCE_VHCI_H_ +#define _APPLE_BCE_VHCI_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "apple_bce.h" + +/* Forward declaration -- full USB headers included only in .c file */ +struct usb_bus; +struct usb_device; +struct usb_xfer; + +/* + * VHCI limits. + */ +#define BCE_VHCI_MAX_PORTS 16 +#define BCE_VHCI_MAX_DEVICES 32 +#define BCE_VHCI_MAX_ENDPOINTS 32 + +/* Queue element counts */ +#define BCE_VHCI_MSG_QUEUE_EL 32 +#define BCE_VHCI_EVT_QUEUE_EL 256 +#define BCE_VHCI_EVT_PENDING 32 +#define BCE_VHCI_TQ_EL 32 /* Transfer queue elements */ +#define BCE_VHCI_XFER_BUFSZ 4096 /* DMA buffer per transfer queue */ + +/* + * VHCI message format (16 bytes, matches firmware protocol). + */ +struct bce_vhci_message { + uint16_t cmd; + uint16_t status; + uint32_t param1; + uint64_t param2; +} __packed; + +/* + * VHCI command IDs. + */ +enum bce_vhci_cmd_id { + /* Controller commands */ + BCE_VHCI_CMD_CONTROLLER_ENABLE = 0x0001, + BCE_VHCI_CMD_CONTROLLER_DISABLE = 0x0002, + BCE_VHCI_CMD_CONTROLLER_START = 0x0003, + BCE_VHCI_CMD_CONTROLLER_PAUSE = 0x0004, + + /* Port commands */ + BCE_VHCI_CMD_PORT_POWER_ON = 0x0010, + BCE_VHCI_CMD_PORT_POWER_OFF = 0x0011, + BCE_VHCI_CMD_PORT_RESUME = 0x0012, + BCE_VHCI_CMD_PORT_SUSPEND = 0x0013, + BCE_VHCI_CMD_PORT_RESET = 0x0014, + BCE_VHCI_CMD_PORT_DISABLE = 0x0015, + BCE_VHCI_CMD_PORT_STATUS = 0x0016, + BCE_VHCI_CMD_PORT_STATUS_CHANGE = 0x0018, + + /* Device commands */ + BCE_VHCI_CMD_DEVICE_CREATE = 0x0030, + BCE_VHCI_CMD_DEVICE_DESTROY = 0x0031, + + /* Endpoint commands */ + BCE_VHCI_CMD_ENDPOINT_CREATE = 0x0040, + BCE_VHCI_CMD_ENDPOINT_DESTROY = 0x0041, + BCE_VHCI_CMD_ENDPOINT_SET_STATE = 0x0042, + BCE_VHCI_CMD_ENDPOINT_REQ_STATE = 0x0043, + BCE_VHCI_CMD_ENDPOINT_RESET = 0x0044, + + /* Transfer commands */ + BCE_VHCI_CMD_TRANSFER_REQUEST = 0x1000, + BCE_VHCI_CMD_CTRL_TRANSFER_STATUS = 0x1005, + + /* Reply flag -- firmware replies have cmd | 0x8000 */ + BCE_VHCI_CMD_REPLY_FLAG = 0x8000, + + /* Cancel flag -- timeout sends cmd | 0x4000 */ + BCE_VHCI_CMD_CANCEL_FLAG = 0x4000, +}; + +/* + * VHCI message status codes. + */ +enum bce_vhci_msg_status { + BCE_VHCI_SUCCESS = 1, + BCE_VHCI_ERROR = 2, + BCE_VHCI_PIPE_STALL = 3, + BCE_VHCI_ABORT = 4, + BCE_VHCI_BAD_ARGUMENT = 5, + BCE_VHCI_OVERRUN = 6, + BCE_VHCI_INTERNAL_ERROR = 7, + BCE_VHCI_NO_POWER = 8, + BCE_VHCI_UNSUPPORTED = 9, +}; + +/* + * Endpoint states. + */ +enum bce_vhci_endpoint_state { + BCE_VHCI_ENDP_ACTIVE = 0, + BCE_VHCI_ENDP_PAUSED = 1, + BCE_VHCI_ENDP_STALLED = 2, +}; + +/* + * Control transfer state machine. + */ +enum bce_vhci_ctrl_state { + BCE_VHCI_CTRL_IDLE = 0, + BCE_VHCI_CTRL_SETUP = 1, /* Awaiting setup XFER_REQ */ + BCE_VHCI_CTRL_DATA = 2, /* Awaiting data XFER_REQ */ + BCE_VHCI_CTRL_STATUS = 3, /* Awaiting CTRL_XFER_STATUS */ +}; + +/* + * Pause sources (bitmask). + */ +#define BCE_VHCI_PAUSE_INTERNAL 0x01 +#define BCE_VHCI_PAUSE_FIRMWARE 0x02 +#define BCE_VHCI_PAUSE_SUSPEND 0x04 +#define BCE_VHCI_PAUSE_SHUTDOWN 0x08 + +/* + * Port status bit mapping (firmware -> USB). + * Firmware uses its own bit encoding; we translate in roothub_exec. + */ +#define BCE_VHCI_PORT_CONNECTED 0x0004 +#define BCE_VHCI_PORT_ENABLED 0x0010 +#define BCE_VHCI_PORT_SUSPENDED 0x0060 +#define BCE_VHCI_PORT_OVERCURRENT 0x0002 +#define BCE_VHCI_PORT_RESET 0x0008 +#define BCE_VHCI_PORT_C_CONNECTION 0x40000 + +/* + * VHCI message queue (host -> device). + */ +struct bce_vhci_msg_queue { + struct bce_queue_cq *cq; + struct bce_queue_sq *sq; + bus_dma_tag_t dma_tag; + bus_dmamap_t dma_map; + bus_addr_t dma_addr; + struct bce_vhci_message *data; + uint32_t el_count; +}; + +/* + * VHCI event queue (device -> host). + * Single contiguous DMA buffer for all receive slots. + */ +struct bce_vhci_evt_queue { + struct bce_queue_sq *sq; + bus_dma_tag_t dma_tag; + bus_dmamap_t dma_map; + bus_addr_t dma_addr; + struct bce_vhci_message *data; + uint32_t el_count; + void *userdata; +}; + +/* + * VHCI command queue (synchronous command execution). + */ +struct bce_vhci_cmd_queue { + struct bce_vhci_msg_queue *msg; + struct sx exec_lock; /* Serialize callers */ + struct mtx lock; + struct sema completion; + struct bce_vhci_message response; + volatile int pending; + uint16_t expected_cmd; /* Filter late replies */ +}; + +/* + * VHCI transfer queue (per endpoint). + * + * Each endpoint gets a CQ + IN SQ + OUT SQ triplet registered with + * firmware as named DMA queues. The DMA buffer is used to shuttle + * USB payloads between the USB stack's frame buffers and firmware. + */ +struct bce_vhci_transfer_queue { + struct bce_vhci_softc *vhci; + uint8_t dev_addr; /* firmware device id */ + uint8_t endp_addr; /* USB endpoint address */ + struct mtx lock; /* Protects SQ submission */ + struct bce_queue_cq *cq; + struct bce_queue_sq *sq_in; /* Device -> host */ + struct bce_queue_sq *sq_out; /* Host -> device */ + struct usb_xfer *active_xfer; + uint32_t paused_by; + int active; /* Queues created with FW */ + int stalled; + int dma_inflight; /* DMA pending */ + int create_pending; /* Deferred ep create */ + struct usb_endpoint_descriptor *create_edesc; + struct usb_xfer *create_xfer; /* Deferred xfer */ + + /* DMA buffer for data transfer */ + bus_dma_tag_t dma_tag; + bus_dmamap_t dma_map; + bus_addr_t dma_addr; + void *dma_buf; + + /* Control transfer state machine */ + enum bce_vhci_ctrl_state ctrl_state; + uint8_t ctrl_dir; /* UE_DIR_IN or UE_DIR_OUT */ + uint32_t ctrl_data_len; /* Expected data phase length */ + uint32_t ctrl_actual; /* Actual bytes transferred */ + int ctrl_data_done; /* IN DMA completion seen */ + int ctrl_status_pending; /* Deferred STATUS msg */ + struct bce_vhci_message ctrl_status_msg; /* Saved STATUS for defer */ + + /* Queued transfer waiting for active_xfer to finish */ + struct usb_xfer *pending_xfer; + + /* Deferred firmware event (TRANSFER_REQUEST arrives before xfer) */ + int evt_pending; + struct bce_vhci_message evt_saved; +}; + +/* + * VHCI per-device state. + */ +struct bce_vhci_device { + int allocated; /* Device created with FW */ + uint8_t fw_dev_id; /* Firmware device ID */ + uint8_t port; /* Port number */ + struct bce_vhci_transfer_queue tq[BCE_VHCI_MAX_ENDPOINTS]; +}; + +/* VHCI softc is defined in apple_bce_vhci.c (depends on USB headers) */ +struct bce_vhci_softc; + +/* VHCI driver interface */ +int bce_vhci_attach(struct apple_bce_softc *sc); +int bce_vhci_detach(struct apple_bce_softc *sc); + +#endif /* _APPLE_BCE_VHCI_H_ */ diff --git a/sys/dev/usb/controller/usb_controller.c b/sys/dev/usb/controller/usb_controller.c index 7e89a5ab015..3eb632124b4 100644 --- a/sys/dev/usb/controller/usb_controller.c +++ b/sys/dev/usb/controller/usb_controller.c @@ -135,6 +135,7 @@ DRIVER_MODULE(usbus, octusb, usb_driver, 0, 0); /* Dual Mode Drivers */ DRIVER_MODULE(usbus, dwcotg, usb_driver, 0, 0); +DRIVER_MODULE(usbus, bce_vhci, usb_driver, 0, 0); /*------------------------------------------------------------------------* * usb_probe diff --git a/sys/modules/apple_bce/Makefile b/sys/modules/apple_bce/Makefile index 48dcef114c8..e853c6eec56 100644 --- a/sys/modules/apple_bce/Makefile +++ b/sys/modules/apple_bce/Makefile @@ -2,6 +2,8 @@ KMOD= apple_bce SRCS= apple_bce.c apple_bce_mailbox.c apple_bce_queue.c +SRCS+= apple_bce_vhci.c SRCS+= bus_if.h device_if.h pci_if.h +SRCS+= opt_usb.h opt_bus.h .include