From 6fd2ad9aa39db916bf2da9607653fb133f8fa078 Mon Sep 17 00:00:00 2001 From: Abdelkader Boudih Date: Sun, 14 Jun 2026 13:54:03 -0700 Subject: [PATCH] apple_bce: add Apple T2 Buffer Copy Engine driver DMA ring transport between the host and the T2 coprocessor. Provides mailbox handshake, queue setup, and firmware keepalive for higher-level T2 services (VHCI, audio, etc.). Tested-on: MacBookPro16,2 (A2251), Mac mini 8,1 (A1993) Reviewed by: adrian Differential Revision: https://reviews.freebsd.org/D57088 --- sys/conf/files.amd64 | 3 + sys/dev/apple_bce/apple_bce.c | 642 ++++++++++++++++++++++++++ sys/dev/apple_bce/apple_bce.h | 304 ++++++++++++ sys/dev/apple_bce/apple_bce_mailbox.c | 108 +++++ sys/dev/apple_bce/apple_bce_mailbox.h | 20 + sys/dev/apple_bce/apple_bce_queue.c | 492 ++++++++++++++++++++ sys/dev/apple_bce/apple_bce_queue.h | 52 +++ sys/modules/Makefile | 2 + sys/modules/apple_bce/Makefile | 7 + 9 files changed, 1630 insertions(+) create mode 100644 sys/dev/apple_bce/apple_bce.c create mode 100644 sys/dev/apple_bce/apple_bce.h create mode 100644 sys/dev/apple_bce/apple_bce_mailbox.c create mode 100644 sys/dev/apple_bce/apple_bce_mailbox.h create mode 100644 sys/dev/apple_bce/apple_bce_queue.c create mode 100644 sys/dev/apple_bce/apple_bce_queue.h create mode 100644 sys/modules/apple_bce/Makefile diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index f12547727a9..4d37266d197 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -112,6 +112,9 @@ crypto/openssl/amd64/sha512-x86_64.S optional ossl crypto/openssl/amd64/ossl_aes_gcm_avx512.c optional ossl crypto/openssl/ossl_aes_gcm.c optional ossl dev/amdgpio/amdgpio.c optional amdgpio +dev/apple_bce/apple_bce.c optional apple_bce pci +dev/apple_bce/apple_bce_mailbox.c optional apple_bce pci +dev/apple_bce/apple_bce_queue.c optional apple_bce pci dev/asmc/asmc.c optional asmc isa dev/asmc/asmcmmio.c optional asmc isa dev/axgbe/if_axgbe_pci.c optional axp diff --git a/sys/dev/apple_bce/apple_bce.c b/sys/dev/apple_bce/apple_bce.c new file mode 100644 index 00000000000..8700eddbc0f --- /dev/null +++ b/sys/dev/apple_bce/apple_bce.c @@ -0,0 +1,642 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Apple T2 Buffer Copy Engine (BCE) PCI driver. + * Provides the transport layer for T2 coprocessor communication: + * mailbox handshake, DMA queue setup, and firmware keepalive. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "apple_bce.h" +#include "apple_bce_mailbox.h" +#include "apple_bce_queue.h" + +static int apple_bce_probe(device_t dev); +static int apple_bce_attach(device_t dev); +static int apple_bce_detach(device_t dev); +static void apple_bce_timestamp_cb(void *arg); +static void apple_bce_timestamp_init(struct apple_bce_softc *sc); +static void apple_bce_timestamp_start(struct apple_bce_softc *sc, + int is_initial); +static void apple_bce_timestamp_stop(struct apple_bce_softc *sc); + +/* + * Timestamp protocol -- T2 firmware expects periodic boottime updates. + * + * Register layout at BAR4 + 0xC000: + * regb + 0 (0xC000): high 32 bits / control + * regb + 8 (0xC008): low 32 bits / data + * + * Control opcodes (written to regb+8 with 0xFFFFFFFF to regb+0): + * -4 (0xFFFFFFFC): initial start + * -3 (0xFFFFFFFD): restart + * -2 (0xFFFFFFFE): stop + */ +static void +apple_bce_timestamp_init(struct apple_bce_softc *sc) +{ + /* Read control register and barrier to sync with firmware */ + bus_read_4(sc->sc_bar4, BCE_REG_TIMESTAMP); + mb(); +} + +static void +apple_bce_timestamp_start(struct apple_bce_softc *sc, int is_initial) +{ + /* Send start opcode: -4 for initial, -3 for restart */ + bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8, + is_initial ? 0xFFFFFFFC : 0xFFFFFFFD); + bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP, 0xFFFFFFFF); + + mtx_lock_spin(&sc->sc_timestamp_lock); + sc->sc_timestamp_stopped = 0; + mtx_unlock_spin(&sc->sc_timestamp_lock); + + callout_reset(&sc->sc_timestamp_co, hz * BCE_TIMESTAMP_MS / 1000, + apple_bce_timestamp_cb, sc); +} + +static void +apple_bce_timestamp_stop(struct apple_bce_softc *sc) +{ + mtx_lock_spin(&sc->sc_timestamp_lock); + sc->sc_timestamp_stopped = 1; + mtx_unlock_spin(&sc->sc_timestamp_lock); + + callout_drain(&sc->sc_timestamp_co); + + /* Send stop opcode */ + bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8, 0xFFFFFFFE); + bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP, 0xFFFFFFFF); +} + +static void +apple_bce_timestamp_cb(void *arg) +{ + struct apple_bce_softc *sc = arg; + struct bintime bt; + uint64_t ns; + + /* Read to sync, then barrier */ + bus_read_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8); + mb(); + + /* Get boot time in nanoseconds */ + binuptime(&bt); + ns = (uint64_t)bt.sec * 1000000000ULL + + (((uint64_t)(bt.frac >> 32) * 1000000000ULL) >> 32); + + /* Write: low 32 bits to regb+8, high 32 bits to regb+0 */ + bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8, (uint32_t)ns); + bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP, (uint32_t)(ns >> 32)); + + mtx_lock_spin(&sc->sc_timestamp_lock); + if (!sc->sc_timestamp_stopped) + callout_schedule(&sc->sc_timestamp_co, + hz * BCE_TIMESTAMP_MS / 1000); + mtx_unlock_spin(&sc->sc_timestamp_lock); +} + +/* + * IRQ handlers. + */ +static void +apple_bce_mbox_intr(void *arg) +{ + struct apple_bce_softc *sc = arg; + + bce_mailbox_handle_interrupt(&sc->sc_mbox); +} + +static void +apple_bce_dma_intr(void *arg) +{ + struct apple_bce_softc *sc = arg; + int i; + + /* + * Only process registered CQs. sc_cq_list[] contains only CQ + * pointers -- separate from sc_queues[] which mixes CQ and SQ. + */ + mtx_lock(&sc->sc_queues_lock); + for (i = 0; i < BCE_MAX_CQ_COUNT; i++) { + if (sc->sc_cq_list[i] != NULL) + bce_handle_cq_completions(sc, sc->sc_cq_list[i]); + } + mtx_unlock(&sc->sc_queues_lock); +} + +/* + * Enable bus master on PCI function 0 (NVMe). + * T2 requires function 0 to be bus master for DMA on function 1. + */ +static int +apple_bce_enable_pci0_busmaster(device_t dev) +{ + device_t pci0; + device_t bus; + + bus = device_get_parent(dev); + if (bus == NULL) + return (ENXIO); + + /* + * Find function 0 on the same bus/slot. + * Our device is function 1; function 0 is NVMe. + */ + pci0 = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev), + pci_get_slot(dev), 0); + if (pci0 == NULL) { + device_printf(dev, "cannot find PCI function 0\n"); + return (ENXIO); + } + + pci_enable_busmaster(pci0); + device_printf(dev, "enabled bus master on function 0 (%s)\n", + device_get_nameunit(pci0)); + return (0); +} + +/* + * Firmware handshake via mailbox. + */ +static int +apple_bce_fw_handshake(struct apple_bce_softc *sc) +{ + uint64_t reply; + int error; + + error = bce_mailbox_send(&sc->sc_mbox, + BCE_MB_MSG(BCE_MB_SET_FW_PROTOCOL_VER, BCE_FW_PROTOCOL_VER), + &reply, BCE_MBOX_TIMEOUT_MS); + if (error != 0) { + device_printf(sc->sc_dev, "firmware handshake timeout\n"); + return (error); + } + + if (BCE_MB_TYPE(reply) != BCE_MB_SET_FW_PROTOCOL_VER || + BCE_MB_VALUE(reply) != BCE_FW_PROTOCOL_VER) { + device_printf(sc->sc_dev, + "firmware version mismatch: got type=%u val=0x%llx\n", + BCE_MB_TYPE(reply), + (unsigned long long)BCE_MB_VALUE(reply)); + return (ENODEV); + } + + device_printf(sc->sc_dev, "firmware handshake OK (protocol 0x%x)\n", + BCE_FW_PROTOCOL_VER); + return (0); +} + +/* + * DMA callback for command queue registration. + */ +static void +bce_reg_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + struct { bus_addr_t addr; int error; } *cb = arg; + + cb->error = error; + if (error == 0) + cb->addr = segs[0].ds_addr; +} + +/* + * Register a command queue (CQ or SQ) with firmware via mailbox. + * The memcfg struct is DMA-mapped and its physical address sent + * in the mailbox message -- firmware reads the config from DMA. + */ +static int +apple_bce_register_cmd_queue(struct apple_bce_softc *sc, + struct bce_queue_memcfg *cfg, int is_sq) +{ + bus_dma_tag_t tag; + bus_dmamap_t map; + bus_addr_t paddr; + struct bce_queue_memcfg *dma_cfg; + struct { + bus_addr_t addr; + int error; + } cb; + uint64_t reply; + int error, cmd_type; + + /* Allocate DMA-coherent buffer for memcfg (8-byte aligned) */ + error = bus_dma_tag_create(sc->sc_dma_tag, + 8, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, + NULL, NULL, sizeof(*cfg), 1, sizeof(*cfg), + BUS_DMA_WAITOK, NULL, NULL, &tag); + if (error != 0) + return (error); + + error = bus_dmamem_alloc(tag, (void **)&dma_cfg, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &map); + if (error != 0) { + bus_dma_tag_destroy(tag); + return (error); + } + + /* Copy config and load DMA address */ + memcpy(dma_cfg, cfg, sizeof(*cfg)); + + cb.error = 0; + error = bus_dmamap_load(tag, map, dma_cfg, sizeof(*cfg), + bce_reg_dma_cb, &cb, BUS_DMA_WAITOK); + if (error != 0 || cb.error != 0) { + bus_dmamem_free(tag, dma_cfg, map); + bus_dma_tag_destroy(tag); + return (error != 0 ? error : cb.error); + } + paddr = cb.addr; + + /* Sync DMA buffer before device access */ + bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE); + + /* Send DMA address of memcfg to firmware */ + cmd_type = is_sq ? BCE_MB_REGISTER_CMD_SQ : BCE_MB_REGISTER_CMD_CQ; + error = bce_mailbox_send(&sc->sc_mbox, + BCE_MB_MSG(cmd_type, paddr), &reply, BCE_MBOX_TIMEOUT_MS); + + bus_dmamap_sync(tag, map, BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(tag, map); + bus_dmamem_free(tag, dma_cfg, map); + bus_dma_tag_destroy(tag); + + if (error != 0) + return (error); + + if (BCE_MB_TYPE(reply) != BCE_MB_REGISTER_QUEUE_REPLY) { + device_printf(sc->sc_dev, + "unexpected queue registration reply: type=%u\n", + BCE_MB_TYPE(reply)); + return (EINVAL); + } + + return (0); +} + +/* + * Setup command queues (CQ qid=0, SQ qid=1). + */ +static int +apple_bce_setup_cmd_queues(struct apple_bce_softc *sc) +{ + struct bce_queue_memcfg cfg; + struct bce_queue_sq *sq; + int error; + + /* Allocate command CQ (qid 0, 32 entries) */ + sc->sc_cmd_cq = bce_alloc_cq(sc, 0, 32); + if (sc->sc_cmd_cq == NULL) { + device_printf(sc->sc_dev, "failed to allocate command CQ\n"); + return (ENOMEM); + } + + /* Register CQ with firmware via DMA-mapped memcfg */ + bce_get_cq_memcfg(sc->sc_cmd_cq, &cfg); + error = apple_bce_register_cmd_queue(sc, &cfg, 0); + if (error != 0) { + device_printf(sc->sc_dev, "failed to register command CQ\n"); + goto fail_cq; + } + + /* Store CQ in queue registries */ + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[0] = sc->sc_cmd_cq; + sc->sc_cq_list[0] = sc->sc_cmd_cq; + mtx_unlock(&sc->sc_queues_lock); + + /* Allocate command SQ (qid 1, 64-byte elements, 32 entries) */ + sq = bce_alloc_sq(sc, 1, BCE_CMD_SIZE, 32, NULL, NULL); + if (sq == NULL) { + device_printf(sc->sc_dev, "failed to allocate command SQ\n"); + goto fail_cq; + } + + /* Wrap SQ in command queue (sets completion callback internally) */ + sc->sc_cmd_cmdq = bce_alloc_cmdq(sc, sq); + if (sc->sc_cmd_cmdq == NULL) { + bce_free_sq(sc, sq); + goto fail_cq; + } + + /* Register SQ with firmware via DMA-mapped memcfg */ + bce_get_sq_memcfg(sq, sc->sc_cmd_cq, &cfg); + error = apple_bce_register_cmd_queue(sc, &cfg, 1); + if (error != 0) { + device_printf(sc->sc_dev, "failed to register command SQ\n"); + goto fail_sq; + } + + /* Store SQ in queue registry */ + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[1] = sq; + sc->sc_int_sq_list[0] = sq; + mtx_unlock(&sc->sc_queues_lock); + + device_printf(sc->sc_dev, "command queues created (CQ=0, SQ=1)\n"); + return (0); + +fail_sq: + bce_free_cmdq(sc->sc_cmd_cmdq); + sc->sc_cmd_cmdq = NULL; + bce_free_sq(sc, sq); +fail_cq: + /* Clear CQ from registries before freeing */ + mtx_lock(&sc->sc_queues_lock); + sc->sc_queues[0] = NULL; + sc->sc_cq_list[0] = NULL; + mtx_unlock(&sc->sc_queues_lock); + bce_free_cq(sc, sc->sc_cmd_cq); + sc->sc_cmd_cq = NULL; + return (error != 0 ? error : ENOMEM); +} + +/* + * PCI probe. + */ +static int +apple_bce_probe(device_t dev) +{ + if (pci_get_vendor(dev) != BCE_PCI_VENDOR_APPLE || + pci_get_device(dev) != BCE_PCI_DEVICE_T2) + return (ENXIO); + + /* Only attach to function 1 (BCE), not function 0 (NVMe) */ + if (pci_get_function(dev) != 1) + return (ENXIO); + + device_set_desc(dev, "Apple T2 Buffer Copy Engine"); + return (BUS_PROBE_DEFAULT); +} + +/* + * PCI attach. + */ +static int +apple_bce_attach(device_t dev) +{ + struct apple_bce_softc *sc = device_get_softc(dev); + int error; + + sc->sc_dev = dev; + pci_enable_busmaster(dev); + mtx_init(&sc->sc_queues_lock, "bce_queues", NULL, MTX_DEF); + mtx_init(&sc->sc_timestamp_lock, "bce_ts", NULL, MTX_SPIN); + callout_init(&sc->sc_timestamp_co, 1); + + /* Map BAR2 (DMA) and BAR4 (mailbox) */ + sc->sc_bar2_rid = PCIR_BAR(2); + sc->sc_bar2 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &sc->sc_bar2_rid, RF_ACTIVE); + if (sc->sc_bar2 == NULL) { + device_printf(dev, "cannot map BAR2 (DMA)\n"); + error = ENXIO; + goto fail; + } + + sc->sc_bar4_rid = PCIR_BAR(4); + sc->sc_bar4 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &sc->sc_bar4_rid, RF_ACTIVE); + if (sc->sc_bar4 == NULL) { + device_printf(dev, "cannot map BAR4 (mailbox)\n"); + error = ENXIO; + goto fail; + } + + /* Allocate MSI vectors (need at least 5: mbox=0, dma=4) */ + sc->sc_msi_count = 8; /* Must be power of 2 for FreeBSD */ + error = pci_alloc_msi(dev, &sc->sc_msi_count); + if (error != 0 || sc->sc_msi_count < 8) { + device_printf(dev, "cannot allocate MSI vectors: " + "error=%d got=%d\n", error, sc->sc_msi_count); + if (error == 0 && sc->sc_msi_count > 0) + pci_release_msi(dev); + sc->sc_msi_count = 0; + error = ENXIO; + goto fail; + } + + /* Initialize mailbox before installing handlers */ + bce_mailbox_init(&sc->sc_mbox, sc->sc_bar4); + + /* Setup IRQ: vector 0 = mailbox, vector 4 = DMA */ + sc->sc_irq_rid_mbox = 1; /* MSI vectors start at rid 1 */ + sc->sc_irq_mbox = bus_alloc_resource_any(dev, SYS_RES_IRQ, + &sc->sc_irq_rid_mbox, RF_ACTIVE); + if (sc->sc_irq_mbox == NULL) { + device_printf(dev, "cannot allocate mailbox IRQ\n"); + error = ENXIO; + goto fail; + } + + error = bus_setup_intr(dev, sc->sc_irq_mbox, + INTR_TYPE_MISC | INTR_MPSAFE, + NULL, apple_bce_mbox_intr, sc, &sc->sc_irq_mbox_cookie); + if (error != 0) { + device_printf(dev, "cannot setup mailbox IRQ\n"); + goto fail; + } + + sc->sc_irq_rid_dma = 5; /* Vector 4 = rid 5 */ + sc->sc_irq_dma = bus_alloc_resource_any(dev, SYS_RES_IRQ, + &sc->sc_irq_rid_dma, RF_ACTIVE); + if (sc->sc_irq_dma == NULL) { + device_printf(dev, "cannot allocate DMA IRQ\n"); + error = ENXIO; + goto fail; + } + + error = bus_setup_intr(dev, sc->sc_irq_dma, + INTR_TYPE_MISC | INTR_MPSAFE, + NULL, apple_bce_dma_intr, sc, &sc->sc_irq_dma_cookie); + if (error != 0) { + device_printf(dev, "cannot setup DMA IRQ\n"); + goto fail; + } + + /* Create parent DMA tag with 37-bit addressing limit */ + error = bus_dma_tag_create(bus_get_dma_tag(dev), + 1, 0, /* alignment, boundary */ + (1ULL << 37) - 1, /* lowaddr: 37-bit limit */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter */ + BUS_SPACE_MAXSIZE, /* maxsize */ + BUS_SPACE_UNRESTRICTED, /* nsegments */ + BUS_SPACE_MAXSIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc */ + &sc->sc_dma_tag); + if (error != 0) { + device_printf(dev, "cannot create DMA tag\n"); + goto fail; + } + + /* + * Enable bus master on function 0 (NVMe) -- T2 quirk. + * Must be done before firmware handshake; T2 rejects DMA + * on function 1 unless function 0 is bus master. + */ + apple_bce_enable_pci0_busmaster(dev); + + /* Initialize and start timestamp keepalive */ + apple_bce_timestamp_init(sc); + apple_bce_timestamp_start(sc, 1); + + /* Firmware handshake */ + error = apple_bce_fw_handshake(sc); + if (error != 0) + goto fail; + + /* Setup command queues */ + error = apple_bce_setup_cmd_queues(sc); + if (error != 0) + goto fail; + + device_printf(dev, "Apple T2 BCE initialized\n"); + return (0); + +fail: + apple_bce_detach(dev); + return (error); +} + +/* + * PCI detach. + */ +static int +apple_bce_detach(device_t dev) +{ + struct apple_bce_softc *sc = device_get_softc(dev); + + /* 1. Stop timestamp */ + if (sc->sc_bar4 != NULL && mtx_initialized(&sc->sc_timestamp_lock)) + apple_bce_timestamp_stop(sc); + else + callout_drain(&sc->sc_timestamp_co); + + /* 2. Tear down IRQs first -- no more interrupts after this */ + if (sc->sc_irq_dma_cookie != NULL) { + bus_teardown_intr(dev, sc->sc_irq_dma, sc->sc_irq_dma_cookie); + sc->sc_irq_dma_cookie = NULL; + } + if (sc->sc_irq_mbox_cookie != NULL) { + bus_teardown_intr(dev, sc->sc_irq_mbox, + sc->sc_irq_mbox_cookie); + sc->sc_irq_mbox_cookie = NULL; + } + + /* 3. Free command queues (safe now -- no IRQs can fire) */ + if (sc->sc_cmd_cmdq != NULL) { + struct bce_queue_sq *cmd_sq; + + cmd_sq = sc->sc_cmd_cmdq->sq; + bce_free_cmdq(sc->sc_cmd_cmdq); + sc->sc_cmd_cmdq = NULL; + bce_free_sq(sc, cmd_sq); + } + if (sc->sc_cmd_cq != NULL) { + bce_free_cq(sc, sc->sc_cmd_cq); + sc->sc_cmd_cq = NULL; + } + + /* 4. Clear queue registries */ + if (mtx_initialized(&sc->sc_queues_lock)) { + mtx_lock(&sc->sc_queues_lock); + memset(sc->sc_queues, 0, sizeof(sc->sc_queues)); + memset(sc->sc_cq_list, 0, sizeof(sc->sc_cq_list)); + memset(sc->sc_int_sq_list, 0, sizeof(sc->sc_int_sq_list)); + mtx_unlock(&sc->sc_queues_lock); + } + + /* 5. Destroy mailbox */ + bce_mailbox_destroy(&sc->sc_mbox); + + /* 6. Release DMA tag */ + if (sc->sc_dma_tag != NULL) { + bus_dma_tag_destroy(sc->sc_dma_tag); + sc->sc_dma_tag = NULL; + } + + /* 7. Release IRQ resources */ + if (sc->sc_irq_dma != NULL) { + bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irq_rid_dma, + sc->sc_irq_dma); + sc->sc_irq_dma = NULL; + } + if (sc->sc_irq_mbox != NULL) { + bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irq_rid_mbox, + sc->sc_irq_mbox); + sc->sc_irq_mbox = NULL; + } + if (sc->sc_msi_count > 0) { + pci_release_msi(dev); + sc->sc_msi_count = 0; + } + + /* 8. Release BARs */ + if (sc->sc_bar4 != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_bar4_rid, + sc->sc_bar4); + sc->sc_bar4 = NULL; + } + if (sc->sc_bar2 != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_bar2_rid, + sc->sc_bar2); + sc->sc_bar2 = NULL; + } + + if (mtx_initialized(&sc->sc_timestamp_lock)) + mtx_destroy(&sc->sc_timestamp_lock); + if (mtx_initialized(&sc->sc_queues_lock)) + mtx_destroy(&sc->sc_queues_lock); + + return (0); +} + +static device_method_t apple_bce_methods[] = { + DEVMETHOD(device_probe, apple_bce_probe), + DEVMETHOD(device_attach, apple_bce_attach), + DEVMETHOD(device_detach, apple_bce_detach), + DEVMETHOD_END +}; + +static driver_t apple_bce_driver = { + "apple_bce", + apple_bce_methods, + sizeof(struct apple_bce_softc) +}; + +DRIVER_MODULE(apple_bce, pci, apple_bce_driver, NULL, NULL); +MODULE_DEPEND(apple_bce, pci, 1, 1, 1); +static const struct { + uint16_t vendor; + uint16_t device; +} apple_bce_pnp[] = { + { BCE_PCI_VENDOR_APPLE, BCE_PCI_DEVICE_T2 }, +}; + +MODULE_PNP_INFO("U16:vendor;U16:device", pci, apple_bce, apple_bce_pnp, + nitems(apple_bce_pnp)); diff --git a/sys/dev/apple_bce/apple_bce.h b/sys/dev/apple_bce/apple_bce.h new file mode 100644 index 00000000000..7c2198c7232 --- /dev/null +++ b/sys/dev/apple_bce/apple_bce.h @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Apple T2 Buffer Copy Engine (BCE) driver. + * PCI transport layer for T2 coprocessor communication. + */ + +#ifndef _APPLE_BCE_H_ +#define _APPLE_BCE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BCE_PCI_VENDOR_APPLE 0x106b +#define BCE_PCI_DEVICE_T2 0x1801 + +#define BCE_MAX_QUEUE_COUNT 0x100 +#define BCE_MAX_CQ_COUNT 16 /* Max completion queues tracked */ +#define BCE_QUEUE_USER_MIN 2 +#define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1) +#define BCE_CMD_SIZE 0x40 + +#define BCE_FW_PROTOCOL_VER 0x20001 +#define BCE_MBOX_TIMEOUT_MS 1000 +#define BCE_TIMESTAMP_MS 150 /* Firmware keepalive interval */ + +/* + * Mailbox register offsets (BAR4). + */ +#define BCE_REG_MBOX_OUT 0x820 /* 4x u32 to-device */ +#define BCE_REG_MBOX_REPLY 0x810 /* 4x u32 from-device */ +#define BCE_REG_MBOX_REPLY_CTR 0x108 /* Reply counter (bits 23:20 = count) */ +#define BCE_REG_TIMESTAMP 0xC000 /* Timestamp sync */ + +/* + * DMA register offsets (BAR2). + */ +#define BCE_REG_DOORBELL_BASE 0x44000 + +/* + * Mailbox message format: 6-bit type (63:58) + 58-bit value. + */ +#define BCE_MB_MSG(type, val) (((uint64_t)(type) << 58) | \ + ((val) & 0x3FFFFFFFFFFFFFFULL)) +#define BCE_MB_TYPE(v) ((uint32_t)((v) >> 58)) +#define BCE_MB_VALUE(v) ((v) & 0x3FFFFFFFFFFFFFFULL) + +/* + * Mailbox message types. + */ +enum bce_mb_type { + BCE_MB_REGISTER_CMD_SQ = 0x07, + BCE_MB_REGISTER_CMD_CQ = 0x08, + BCE_MB_REGISTER_QUEUE_REPLY = 0x0B, + BCE_MB_SET_FW_PROTOCOL_VER = 0x0C, + BCE_MB_SLEEP_NO_STATE = 0x14, + BCE_MB_RESTORE_NO_STATE = 0x15, + BCE_MB_SAVE_AND_SLEEP = 0x17, + BCE_MB_RESTORE_AND_WAKE = 0x18, + BCE_MB_SAVE_FAILURE = 0x19, + BCE_MB_SAVE_RESTORE_COMPLETE = 0x1A, +}; + +/* + * Queue types. + */ +enum bce_queue_type { + BCE_QUEUE_CQ = 0, + BCE_QUEUE_SQ = 1, +}; + +/* + * Completion queue entry. + */ +struct bce_qe_completion { + uint64_t result; + uint64_t data_size; + uint16_t qid; /* Source SQ */ + uint16_t completion_index; + uint16_t status; + uint16_t flags; +#define BCE_CQ_FLAG_PENDING 0x8000 +} __packed; + +/* + * Completion status codes. + */ +enum bce_completion_status { + BCE_COMP_SUCCESS = 0, + BCE_COMP_ERROR = 1, + BCE_COMP_ABORTED = 2, + BCE_COMP_NO_SPACE = 3, + BCE_COMP_OVERRUN = 4, +}; + +/* + * Submission queue entry. + */ +struct bce_qe_submission { + uint64_t length; + uint64_t addr; + uint64_t segl_addr; /* Scatter-gather list */ + uint64_t segl_length; +} __packed; + +/* + * Queue memory configuration (for mailbox registration). + */ +struct bce_queue_memcfg { + uint16_t qid; + uint16_t el_count; + uint16_t vector_or_cq; + uint16_t _pad; + uint64_t addr; + uint64_t length; +} __packed; + +/* + * Completion data cached per SQ slot. + */ +struct bce_sq_completion_data { + uint32_t status; + uint64_t data_size; + uint64_t result; +}; + +/* + * Completion callback. + */ +struct bce_queue_sq; +typedef void (*bce_sq_completion_fn)(struct bce_queue_sq *sq); + +/* + * Completion queue. + */ +struct bce_queue_cq { + int qid; + uint32_t el_count; + bus_dma_tag_t dma_tag; + bus_dmamap_t dma_map; + bus_addr_t dma_addr; + struct bce_qe_completion *data; + uint32_t index; /* Consumer index */ +}; + +/* + * Submission queue. + */ +struct bce_queue_sq { + int qid; + uint32_t el_size; + uint32_t el_count; + bus_dma_tag_t dma_tag; + bus_dmamap_t dma_map; + bus_addr_t dma_addr; + void *data; + void *userdata; + + /* Submission tracking */ + volatile int available_commands; + uint32_t head; + uint32_t tail; + + /* Completion tracking */ + uint32_t completion_cidx; + uint32_t completion_tail; + struct bce_sq_completion_data *completion_data; + int has_pending; + bce_sq_completion_fn completion; +}; + +/* + * Command queue (wraps SQ for synchronous ops). + */ +struct bce_queue_cmdq_result { + struct sema cmpl; + uint32_t status; + uint64_t result; +}; + +struct bce_queue_cmdq { + struct bce_queue_sq *sq; + struct mtx lck; + struct bce_queue_cmdq_result **tres; /* Pending results per slot */ +}; + +/* + * Command queue command IDs. + */ +enum bce_cmdq_cmd { + BCE_CMD_REGISTER_QUEUE = 0x20, + BCE_CMD_UNREGISTER_QUEUE = 0x30, + BCE_CMD_FLUSH_QUEUE = 0x40, + BCE_CMD_SET_QUEUE_PROPERTY = 0x50, +}; + +/* + * Register queue command payload (fits in BCE_CMD_SIZE). + */ +struct bce_cmdq_reg_cmd { + uint16_t cmd; + uint16_t flags; +#define BCE_CMDQ_FLAG_NAMED 0x02 +#define BCE_CMDQ_FLAG_OUT 0x01 + uint16_t qid; + uint16_t _pad; + uint16_t el_count; + uint16_t vector_or_cq; + uint16_t _pad2; + uint16_t name_len; + char name[0x20]; + uint64_t addr; + uint64_t length; +} __packed; + +/* + * Simple queue command (unregister/flush). + */ +struct bce_cmdq_simple_cmd { + uint16_t cmd; + uint16_t flags; + uint16_t qid; +} __packed; + +/* + * Mailbox state. + */ +struct bce_mailbox { + struct resource *reg; /* BAR4 resource */ + volatile int status; /* 0=idle, 1=pending, 2=ready */ + int initialized; + struct sema mb_cmpl; + uint64_t mb_result; +}; + +/* + * Main device softc. + */ +struct apple_bce_softc { + device_t sc_dev; + + /* PCI resources */ + int sc_bar2_rid; + struct resource *sc_bar2; /* DMA registers */ + int sc_bar4_rid; + struct resource *sc_bar4; /* Mailbox registers */ + + /* MSI */ + int sc_msi_count; + int sc_irq_rid_mbox; + struct resource *sc_irq_mbox; + void *sc_irq_mbox_cookie; + int sc_irq_rid_dma; + struct resource *sc_irq_dma; + void *sc_irq_dma_cookie; + + /* DMA */ + bus_dma_tag_t sc_dma_tag; + + /* Mailbox */ + struct bce_mailbox sc_mbox; + + /* Timestamp keepalive */ + struct callout sc_timestamp_co; + struct mtx sc_timestamp_lock; + int sc_timestamp_stopped; + + /* Command path */ + struct bce_queue_cq *sc_cmd_cq; + struct bce_queue_cmdq *sc_cmd_cmdq; + + /* Queue registry */ + void *sc_queues[BCE_MAX_QUEUE_COUNT]; + struct mtx sc_queues_lock; + struct bce_queue_cq *sc_cq_list[BCE_MAX_CQ_COUNT]; + struct bce_queue_sq *sc_int_sq_list[BCE_MAX_QUEUE_COUNT]; +}; + +/* Inline helpers */ +static __inline void * +bce_sq_element(struct bce_queue_sq *sq, uint32_t idx) +{ + return ((char *)sq->data + (idx * sq->el_size)); +} + +static __inline struct bce_qe_completion * +bce_cq_element(struct bce_queue_cq *cq, uint32_t idx) +{ + return (&cq->data[idx]); +} + +#endif /* _APPLE_BCE_H_ */ diff --git a/sys/dev/apple_bce/apple_bce_mailbox.c b/sys/dev/apple_bce/apple_bce_mailbox.c new file mode 100644 index 00000000000..abdb809b5c5 --- /dev/null +++ b/sys/dev/apple_bce/apple_bce_mailbox.c @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Apple BCE mailbox -- 64-bit request/reply over BAR4 registers. + */ + +#include +#include +#include +#include +#include +#include + +#include "apple_bce.h" +#include "apple_bce_mailbox.h" + +int +bce_mailbox_init(struct bce_mailbox *mb, struct resource *reg) +{ + mb->reg = reg; + mb->status = 0; + sema_init(&mb->mb_cmpl, 0, "bce_mb"); + mb->mb_result = 0; + mb->initialized = 1; + return (0); +} + +void +bce_mailbox_destroy(struct bce_mailbox *mb) +{ + if (mb->initialized) { + sema_destroy(&mb->mb_cmpl); + mb->initialized = 0; + } +} + +/* + * Send a mailbox message and wait for reply. + * Only one message in flight at a time (enforced by cmpxchg). + */ +int +bce_mailbox_send(struct bce_mailbox *mb, uint64_t msg, uint64_t *recv, + unsigned int timeout_ms) +{ + int old; + + /* Acquire: idle (0) -> pending (1) */ + old = atomic_cmpset_int(&mb->status, 0, 1); + if (old == 0) + return (EBUSY); + + /* Write 64-bit message as 4x u32 to MBOX_OUT */ + bus_write_4(mb->reg, BCE_REG_MBOX_OUT, (uint32_t)msg); + bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 4, (uint32_t)(msg >> 32)); + bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 8, 0); + bus_write_4(mb->reg, BCE_REG_MBOX_OUT + 12, 0); + + /* Wait for interrupt-driven reply */ + if (sema_timedwait(&mb->mb_cmpl, hz * timeout_ms / 1000) != 0) { + /* Timeout -- reset to idle */ + atomic_store_int(&mb->status, 0); + return (ETIMEDOUT); + } + + if (atomic_load_int(&mb->status) != 2) { + atomic_store_int(&mb->status, 0); + return (ETIMEDOUT); + } + + if (recv != NULL) + *recv = mb->mb_result; + + atomic_store_int(&mb->status, 0); + return (0); +} + +/* + * Called from ISR to retrieve mailbox reply. + */ +int +bce_mailbox_handle_interrupt(struct bce_mailbox *mb) +{ + uint32_t res, lo, hi; + int count; + + res = bus_read_4(mb->reg, BCE_REG_MBOX_REPLY_CTR); + count = (res >> 20) & 0xf; + + if (count == 0) + return (ENOMSG); + + while (count-- > 0) { + lo = bus_read_4(mb->reg, BCE_REG_MBOX_REPLY); + hi = bus_read_4(mb->reg, BCE_REG_MBOX_REPLY + 4); + bus_read_4(mb->reg, BCE_REG_MBOX_REPLY + 8); + bus_read_4(mb->reg, BCE_REG_MBOX_REPLY + 12); + mb->mb_result = ((uint64_t)hi << 32) | lo; + } + + if (atomic_load_int(&mb->status) == 1) { + atomic_store_int(&mb->status, 2); + sema_post(&mb->mb_cmpl); + } + + return (0); +} diff --git a/sys/dev/apple_bce/apple_bce_mailbox.h b/sys/dev/apple_bce/apple_bce_mailbox.h new file mode 100644 index 00000000000..82d2bc4d0c5 --- /dev/null +++ b/sys/dev/apple_bce/apple_bce_mailbox.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Apple BCE mailbox interface. + */ + +#ifndef _APPLE_BCE_MAILBOX_H_ +#define _APPLE_BCE_MAILBOX_H_ + +#include "apple_bce.h" + +int bce_mailbox_init(struct bce_mailbox *mb, struct resource *reg); +void bce_mailbox_destroy(struct bce_mailbox *mb); +int bce_mailbox_send(struct bce_mailbox *mb, uint64_t msg, uint64_t *recv, + unsigned int timeout_ms); +int bce_mailbox_handle_interrupt(struct bce_mailbox *mb); + +#endif /* _APPLE_BCE_MAILBOX_H_ */ diff --git a/sys/dev/apple_bce/apple_bce_queue.c b/sys/dev/apple_bce/apple_bce_queue.c new file mode 100644 index 00000000000..76d47f874cd --- /dev/null +++ b/sys/dev/apple_bce/apple_bce_queue.c @@ -0,0 +1,492 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Apple BCE queue management -- CQ, SQ, and command queue operations. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apple_bce.h" +#include "apple_bce_queue.h" + +static MALLOC_DEFINE(M_BCE, "apple_bce", "Apple BCE driver"); + +#define BCE_CMDQ_TIMEOUT_MS 5000 + +/* + * DMA callback for bus_dmamap_load. + */ +struct bce_dma_cb_arg { + bus_addr_t addr; + int error; +}; + +static int +bce_cmdq_wait(struct bce_queue_cmdq *cmdq, uint32_t slot, + struct bce_queue_cmdq_result *res) +{ + int error; + + error = sema_timedwait(&res->cmpl, hz * BCE_CMDQ_TIMEOUT_MS / 1000); + if (error == 0) + return (0); + + mtx_lock(&cmdq->lck); + if (cmdq->tres[slot] == res) + cmdq->tres[slot] = NULL; + mtx_unlock(&cmdq->lck); + + return (ETIMEDOUT); +} + +static void +bce_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + struct bce_dma_cb_arg *cb = arg; + + cb->error = error; + if (error == 0) + cb->addr = segs[0].ds_addr; +} + +/* + * Allocate a completion queue with DMA-coherent memory. + */ +struct bce_queue_cq * +bce_alloc_cq(struct apple_bce_softc *sc, int qid, uint32_t el_count) +{ + struct bce_queue_cq *cq; + struct bce_dma_cb_arg cb; + int error; + + cq = malloc(sizeof(*cq), M_BCE, M_WAITOK | M_ZERO); + cq->qid = qid; + cq->el_count = el_count; + cq->index = 0; + + error = bus_dma_tag_create(sc->sc_dma_tag, + 4, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter */ + el_count * sizeof(struct bce_qe_completion), + 1, /* nsegments */ + el_count * sizeof(struct bce_qe_completion), + BUS_DMA_WAITOK, /* flags */ + NULL, NULL, /* lockfunc */ + &cq->dma_tag); + if (error) { + free(cq, M_BCE); + return (NULL); + } + + error = bus_dmamem_alloc(cq->dma_tag, (void **)&cq->data, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &cq->dma_map); + if (error) { + bus_dma_tag_destroy(cq->dma_tag); + free(cq, M_BCE); + return (NULL); + } + + error = bus_dmamap_load(cq->dma_tag, cq->dma_map, cq->data, + el_count * sizeof(struct bce_qe_completion), + bce_dma_cb, &cb, BUS_DMA_WAITOK); + if (error || cb.error) { + bus_dmamem_free(cq->dma_tag, cq->data, cq->dma_map); + bus_dma_tag_destroy(cq->dma_tag); + free(cq, M_BCE); + return (NULL); + } + cq->dma_addr = cb.addr; + + return (cq); +} + +void +bce_free_cq(struct apple_bce_softc *sc, struct bce_queue_cq *cq) +{ + if (cq == NULL) + return; + bus_dmamap_unload(cq->dma_tag, cq->dma_map); + bus_dmamem_free(cq->dma_tag, cq->data, cq->dma_map); + bus_dma_tag_destroy(cq->dma_tag); + free(cq, M_BCE); +} + +void +bce_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) +{ + cfg->qid = (uint16_t)cq->qid; + cfg->el_count = (uint16_t)cq->el_count; + cfg->vector_or_cq = 0; + cfg->_pad = 0; + cfg->addr = cq->dma_addr; + cfg->length = cq->el_count * sizeof(struct bce_qe_completion); +} + +/* + * Process completions from a CQ and dispatch to the target SQs. + */ +void +bce_handle_cq_completions(struct apple_bce_softc *sc, struct bce_queue_cq *cq) +{ + struct bce_qe_completion *e; + struct bce_queue_sq *sq; + int i; + bus_dmamap_sync(cq->dma_tag, cq->dma_map, BUS_DMASYNC_POSTREAD); + + e = bce_cq_element(cq, cq->index); + if (!(e->flags & BCE_CQ_FLAG_PENDING)) + return; + + while ((e = bce_cq_element(cq, cq->index))->flags & + BCE_CQ_FLAG_PENDING) { + /* Route completion to target SQ (skip qid 0 which is the CQ) */ + if (e->qid > 0 && e->qid < BCE_MAX_QUEUE_COUNT) { + sq = (struct bce_queue_sq *)sc->sc_queues[e->qid]; + if (sq != NULL && + e->completion_index < sq->el_count) { + sq->completion_data[e->completion_index].status = + e->status; + sq->completion_data[e->completion_index].data_size = + e->data_size; + sq->completion_data[e->completion_index].result = + e->result; + /* Advance tail so completion callback sees it */ + sq->completion_tail = + (e->completion_index + 1) % sq->el_count; + sq->has_pending = 1; + } + } + + e->flags = 0; + cq->index = (cq->index + 1) % cq->el_count; + } + + bus_dmamap_sync(cq->dma_tag, cq->dma_map, BUS_DMASYNC_PREWRITE); + + /* Ring doorbell with updated consumer index */ + bus_write_4(sc->sc_bar2, BCE_REG_DOORBELL_BASE + cq->qid * 4, + cq->index); + + /* Fire callbacks on SQs that received completions */ + for (i = 0; i < BCE_MAX_QUEUE_COUNT; i++) { + sq = sc->sc_int_sq_list[i]; + if (sq != NULL && sq->has_pending) { + sq->has_pending = 0; + if (sq->completion != NULL) + sq->completion(sq); + } + } +} + +/* + * Allocate a submission queue with DMA-coherent memory. + */ +struct bce_queue_sq * +bce_alloc_sq(struct apple_bce_softc *sc, int qid, uint32_t el_size, + uint32_t el_count, bce_sq_completion_fn compl, void *userdata) +{ + struct bce_queue_sq *sq; + struct bce_dma_cb_arg cb; + int error; + + sq = malloc(sizeof(*sq), M_BCE, M_WAITOK | M_ZERO); + sq->qid = qid; + sq->el_size = el_size; + sq->el_count = el_count; + sq->completion = compl; + sq->userdata = userdata; + sq->available_commands = el_count - 1; + + error = bus_dma_tag_create(sc->sc_dma_tag, + 4, 0, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, + NULL, NULL, + el_count * el_size, 1, el_count * el_size, + BUS_DMA_WAITOK, + NULL, NULL, + &sq->dma_tag); + if (error) { + free(sq, M_BCE); + return (NULL); + } + + error = bus_dmamem_alloc(sq->dma_tag, &sq->data, + BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sq->dma_map); + if (error) { + bus_dma_tag_destroy(sq->dma_tag); + free(sq, M_BCE); + return (NULL); + } + + error = bus_dmamap_load(sq->dma_tag, sq->dma_map, sq->data, + el_count * el_size, bce_dma_cb, &cb, BUS_DMA_WAITOK); + if (error || cb.error) { + bus_dmamem_free(sq->dma_tag, sq->data, sq->dma_map); + bus_dma_tag_destroy(sq->dma_tag); + free(sq, M_BCE); + return (NULL); + } + sq->dma_addr = cb.addr; + + sq->completion_data = malloc(sizeof(*sq->completion_data) * el_count, + M_BCE, M_WAITOK | M_ZERO); + + return (sq); +} + +void +bce_free_sq(struct apple_bce_softc *sc, struct bce_queue_sq *sq) +{ + if (sq == NULL) + return; + free(sq->completion_data, M_BCE); + bus_dmamap_unload(sq->dma_tag, sq->dma_map); + bus_dmamem_free(sq->dma_tag, sq->data, sq->dma_map); + bus_dma_tag_destroy(sq->dma_tag); + free(sq, M_BCE); +} + +void +bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, + struct bce_queue_memcfg *cfg) +{ + cfg->qid = (uint16_t)sq->qid; + cfg->el_count = (uint16_t)sq->el_count; + cfg->vector_or_cq = (uint16_t)cq->qid; + cfg->_pad = 0; + cfg->addr = sq->dma_addr; + cfg->length = sq->el_count * sq->el_size; +} + +int +bce_reserve_submission(struct bce_queue_sq *sq) +{ + int old, new; + + do { + old = atomic_load_int(&sq->available_commands); + if (old <= 0) + return (EAGAIN); + new = old - 1; + } while (!atomic_cmpset_int(&sq->available_commands, old, new)); + + return (0); +} + +void * +bce_next_submission(struct bce_queue_sq *sq) +{ + void *ret; + + ret = bce_sq_element(sq, sq->tail); + sq->tail = (sq->tail + 1) % sq->el_count; + return (ret); +} + +void +bce_submit_to_device(struct apple_bce_softc *sc, struct bce_queue_sq *sq) +{ + bus_dmamap_sync(sq->dma_tag, sq->dma_map, BUS_DMASYNC_PREWRITE); + bus_write_4(sc->sc_bar2, BCE_REG_DOORBELL_BASE + sq->qid * 4, + sq->tail); +} + +void +bce_notify_submission_complete(struct bce_queue_sq *sq) +{ + sq->head = (sq->head + 1) % sq->el_count; + atomic_add_int(&sq->available_commands, 1); +} + +/* + * Command queue -- wraps an SQ for synchronous control operations. + */ +static void bce_cmdq_completion(struct bce_queue_sq *sq); + +struct bce_queue_cmdq * +bce_alloc_cmdq(struct apple_bce_softc *sc, struct bce_queue_sq *sq) +{ + struct bce_queue_cmdq *cmdq; + + cmdq = malloc(sizeof(*cmdq), M_BCE, M_WAITOK | M_ZERO); + cmdq->sq = sq; + mtx_init(&cmdq->lck, "bce_cmdq", NULL, MTX_DEF); + cmdq->tres = malloc(sizeof(void *) * sq->el_count, M_BCE, + M_WAITOK | M_ZERO); + + /* Wire up completion callback */ + sq->completion = bce_cmdq_completion; + sq->userdata = cmdq; + + return (cmdq); +} + +void +bce_free_cmdq(struct bce_queue_cmdq *cmdq) +{ + if (cmdq == NULL) + return; + mtx_destroy(&cmdq->lck); + free(cmdq->tres, M_BCE); + free(cmdq, M_BCE); +} + +/* + * Command queue completion callback -- wake waiters. + */ +static void +bce_cmdq_completion(struct bce_queue_sq *sq) +{ + struct bce_queue_cmdq *cmdq = sq->userdata; + struct bce_queue_cmdq_result *res; + + mtx_lock(&cmdq->lck); + while (sq->completion_cidx != sq->completion_tail) { + struct bce_sq_completion_data *cd; + + cd = &sq->completion_data[sq->completion_cidx]; + res = cmdq->tres[sq->completion_cidx]; + if (res != NULL) { + res->status = cd->status; + res->result = cd->result; + sema_post(&res->cmpl); + cmdq->tres[sq->completion_cidx] = NULL; + } + sq->completion_cidx = (sq->completion_cidx + 1) % + sq->el_count; + bce_notify_submission_complete(sq); + } + mtx_unlock(&cmdq->lck); +} + +uint32_t +bce_cmd_register_queue(struct bce_queue_cmdq *cmdq, + struct apple_bce_softc *sc, struct bce_queue_memcfg *cfg, + const char *name, int isdirout) +{ + struct bce_queue_cmdq_result res; + struct bce_cmdq_reg_cmd *cmd; + uint32_t slot; + int error; + + sema_init(&res.cmpl, 0, "bce_cmd"); + + if (bce_reserve_submission(cmdq->sq) != 0) { + sema_destroy(&res.cmpl); + return (EAGAIN); + } + + mtx_lock(&cmdq->lck); + slot = cmdq->sq->tail; + cmdq->tres[slot] = &res; + cmd = bce_next_submission(cmdq->sq); + + memset(cmd, 0, BCE_CMD_SIZE); + cmd->cmd = BCE_CMD_REGISTER_QUEUE; + cmd->flags = (name ? BCE_CMDQ_FLAG_NAMED : 0) | + (isdirout ? BCE_CMDQ_FLAG_OUT : 0); + cmd->qid = cfg->qid; + cmd->el_count = cfg->el_count; + cmd->vector_or_cq = cfg->vector_or_cq; + if (name != NULL) { + cmd->name_len = (uint16_t)MIN(strlen(name), sizeof(cmd->name)); + memcpy(cmd->name, name, cmd->name_len); + } + cmd->addr = cfg->addr; + cmd->length = cfg->length; + + bce_submit_to_device(sc, cmdq->sq); + mtx_unlock(&cmdq->lck); + + error = bce_cmdq_wait(cmdq, slot, &res); + sema_destroy(&res.cmpl); + if (error != 0) + return (error); + + return (res.status); +} + +uint32_t +bce_cmd_unregister_queue(struct bce_queue_cmdq *cmdq, + struct apple_bce_softc *sc, int qid) +{ + struct bce_queue_cmdq_result res; + struct bce_cmdq_simple_cmd *cmd; + uint32_t slot; + int error; + + sema_init(&res.cmpl, 0, "bce_cmd"); + + if (bce_reserve_submission(cmdq->sq) != 0) { + sema_destroy(&res.cmpl); + return (EAGAIN); + } + + mtx_lock(&cmdq->lck); + slot = cmdq->sq->tail; + cmdq->tres[slot] = &res; + cmd = bce_next_submission(cmdq->sq); + + memset(cmd, 0, BCE_CMD_SIZE); + cmd->cmd = BCE_CMD_UNREGISTER_QUEUE; + cmd->qid = (uint16_t)qid; + + bce_submit_to_device(sc, cmdq->sq); + mtx_unlock(&cmdq->lck); + + error = bce_cmdq_wait(cmdq, slot, &res); + sema_destroy(&res.cmpl); + if (error != 0) + return (error); + + return (res.status); +} + +uint32_t +bce_cmd_flush_queue(struct bce_queue_cmdq *cmdq, + struct apple_bce_softc *sc, int qid) +{ + struct bce_queue_cmdq_result res; + struct bce_cmdq_simple_cmd *cmd; + uint32_t slot; + int error; + + sema_init(&res.cmpl, 0, "bce_cmd"); + + if (bce_reserve_submission(cmdq->sq) != 0) { + sema_destroy(&res.cmpl); + return (EAGAIN); + } + + mtx_lock(&cmdq->lck); + slot = cmdq->sq->tail; + cmdq->tres[slot] = &res; + cmd = bce_next_submission(cmdq->sq); + + memset(cmd, 0, BCE_CMD_SIZE); + cmd->cmd = BCE_CMD_FLUSH_QUEUE; + cmd->qid = (uint16_t)qid; + + bce_submit_to_device(sc, cmdq->sq); + mtx_unlock(&cmdq->lck); + + error = bce_cmdq_wait(cmdq, slot, &res); + sema_destroy(&res.cmpl); + if (error != 0) + return (error); + + return (res.status); +} diff --git a/sys/dev/apple_bce/apple_bce_queue.h b/sys/dev/apple_bce/apple_bce_queue.h new file mode 100644 index 00000000000..ffd1e1a199f --- /dev/null +++ b/sys/dev/apple_bce/apple_bce_queue.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2026 Abdelkader Boudih + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Apple BCE queue management. + */ + +#ifndef _APPLE_BCE_QUEUE_H_ +#define _APPLE_BCE_QUEUE_H_ + +#include "apple_bce.h" + +struct apple_bce_softc; + +/* CQ operations */ +struct bce_queue_cq *bce_alloc_cq(struct apple_bce_softc *sc, int qid, + uint32_t el_count); +void bce_free_cq(struct apple_bce_softc *sc, struct bce_queue_cq *cq); +void bce_get_cq_memcfg(struct bce_queue_cq *cq, + struct bce_queue_memcfg *cfg); +void bce_handle_cq_completions(struct apple_bce_softc *sc, + struct bce_queue_cq *cq); + +/* SQ operations */ +struct bce_queue_sq *bce_alloc_sq(struct apple_bce_softc *sc, int qid, + uint32_t el_size, uint32_t el_count, + bce_sq_completion_fn compl, void *userdata); +void bce_free_sq(struct apple_bce_softc *sc, struct bce_queue_sq *sq); +void bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, + struct bce_queue_memcfg *cfg); + +/* Submission helpers */ +int bce_reserve_submission(struct bce_queue_sq *sq); +void *bce_next_submission(struct bce_queue_sq *sq); +void bce_submit_to_device(struct apple_bce_softc *sc, + struct bce_queue_sq *sq); +void bce_notify_submission_complete(struct bce_queue_sq *sq); + +/* Command queue */ +struct bce_queue_cmdq *bce_alloc_cmdq(struct apple_bce_softc *sc, + struct bce_queue_sq *sq); +void bce_free_cmdq(struct bce_queue_cmdq *cmdq); +uint32_t bce_cmd_register_queue(struct bce_queue_cmdq *cmdq, + struct apple_bce_softc *sc, struct bce_queue_memcfg *cfg, + const char *name, int isdirout); +uint32_t bce_cmd_unregister_queue(struct bce_queue_cmdq *cmdq, + struct apple_bce_softc *sc, int qid); +uint32_t bce_cmd_flush_queue(struct bce_queue_cmdq *cmdq, + struct apple_bce_softc *sc, int qid); + +#endif /* _APPLE_BCE_QUEUE_H_ */ diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 13100cd3fe5..972441264fb 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -43,6 +43,7 @@ SUBDIR= \ ${_allwinner} \ ${_armv8crypto} \ ${_armv8_rng} \ + ${_apple_bce} \ ${_asmc} \ ata \ ${_aq} \ @@ -863,6 +864,7 @@ _x86bios= x86bios .if ${MACHINE_CPUARCH} == "amd64" _amdgpio= amdgpio +_apple_bce= apple_bce _asmc= asmc _ccp= ccp _enic= enic diff --git a/sys/modules/apple_bce/Makefile b/sys/modules/apple_bce/Makefile new file mode 100644 index 00000000000..48dcef114c8 --- /dev/null +++ b/sys/modules/apple_bce/Makefile @@ -0,0 +1,7 @@ +.PATH: ${SRCTOP}/sys/dev/apple_bce + +KMOD= apple_bce +SRCS= apple_bce.c apple_bce_mailbox.c apple_bce_queue.c +SRCS+= bus_if.h device_if.h pci_if.h + +.include