From 685666aaf7e832dc5df2363c1ffa72fdbbd436ac Mon Sep 17 00:00:00 2001 From: Konstantin Belousov Date: Mon, 18 Nov 2019 20:56:59 +0000 Subject: [PATCH] bus_dma_dmar_set_buswide(9): KPI to indicate that the whole dmar context should share page tables. Practically it means that dma requests from any device on the bus are translated according to the entries loaded for the bus:0:0 device. KPI requires that the slot and function of the device be 0:0, and that no tags for other devices on the bus were used. The intended use are NTBs which pass TLPs from the downstream to the host with slot:func of the downstream originator. Reviewed and tested by: mav Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D22434 --- sys/x86/include/bus_dma.h | 2 ++ sys/x86/iommu/busdma_dmar.c | 28 +++++++++++++++ sys/x86/iommu/intel_ctx.c | 64 ++++++++++++++++++++++++----------- sys/x86/iommu/intel_dmar.h | 12 +++++++ sys/x86/iommu/intel_drv.c | 22 +++++++++++- sys/x86/iommu/intel_gas.c | 1 + sys/x86/iommu/intel_idpgtbl.c | 1 + sys/x86/iommu/intel_intrmap.c | 1 + sys/x86/iommu/intel_qi.c | 1 + sys/x86/iommu/intel_quirks.c | 1 + sys/x86/iommu/intel_utils.c | 1 + sys/x86/x86/busdma_machdep.c | 9 +++++ 12 files changed, 122 insertions(+), 21 deletions(-) diff --git a/sys/x86/include/bus_dma.h b/sys/x86/include/bus_dma.h index b4da787e9cf..016c0fac85d 100644 --- a/sys/x86/include/bus_dma.h +++ b/sys/x86/include/bus_dma.h @@ -191,5 +191,7 @@ _bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, return (tc->impl->map_complete(dmat, map, segs, nsegs, error)); } +bool bus_dma_dmar_set_buswide(device_t dev); + #endif /* !_X86_BUS_DMA_H_ */ diff --git a/sys/x86/iommu/busdma_dmar.c b/sys/x86/iommu/busdma_dmar.c index af8a0129e6c..1154a55159e 100644 --- a/sys/x86/iommu/busdma_dmar.c +++ b/sys/x86/iommu/busdma_dmar.c @@ -289,6 +289,34 @@ dmar_get_dma_tag(device_t dev, device_t child) return (res); } +bool +bus_dma_dmar_set_buswide(device_t dev) +{ + struct dmar_unit *dmar; + device_t parent; + u_int busno, slot, func; + + parent = device_get_parent(dev); + if (device_get_devclass(parent) != devclass_find("pci")) + return (false); + dmar = dmar_find(dev, bootverbose); + if (dmar == NULL) + return (false); + busno = pci_get_bus(dev); + slot = pci_get_slot(dev); + func = pci_get_function(dev); + if (slot != 0 || func != 0) { + if (bootverbose) { + device_printf(dev, + "dmar%d pci%d:%d:%d requested buswide busdma\n", + dmar->unit, busno, slot, func); + } + return (false); + } + dmar_set_buswide_ctx(dmar, busno); + return (true); +} + static MALLOC_DEFINE(M_DMAR_DMAMAP, "dmar_dmamap", "Intel DMAR DMA Map"); static void dmar_bus_schedule_dmamap(struct dmar_unit *unit, diff --git a/sys/x86/iommu/intel_ctx.c b/sys/x86/iommu/intel_ctx.c index cca489388f0..760f7b94f6c 100644 --- a/sys/x86/iommu/intel_ctx.c +++ b/sys/x86/iommu/intel_ctx.c @@ -67,8 +67,8 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include +#include #include static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); @@ -141,20 +141,9 @@ ctx_tag_init(struct dmar_ctx *ctx, device_t dev) } static void -ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move) +ctx_id_entry_init_one(dmar_ctx_entry_t *ctxp, struct dmar_domain *domain, + vm_page_t ctx_root) { - struct dmar_unit *unit; - struct dmar_domain *domain; - vm_page_t ctx_root; - - domain = ctx->domain; - unit = domain->dmar; - KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), - ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", - unit->unit, pci_get_bus(ctx->ctx_tag.owner), - pci_get_slot(ctx->ctx_tag.owner), - pci_get_function(ctx->ctx_tag.owner), - ctxp->ctx1, ctxp->ctx2)); /* * For update due to move, the store is not atomic. It is * possible that DMAR read upper doubleword, while low @@ -166,16 +155,48 @@ ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move) */ dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) | domain->awlvl); + if (ctx_root == NULL) { + dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); + } else { + dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR | + (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | + DMAR_CTX1_P); + } +} + +static void +ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move, + int busno) +{ + struct dmar_unit *unit; + struct dmar_domain *domain; + vm_page_t ctx_root; + int i; + + domain = ctx->domain; + unit = domain->dmar; + KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), + ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", + unit->unit, busno, pci_get_slot(ctx->ctx_tag.owner), + pci_get_function(ctx->ctx_tag.owner), + ctxp->ctx1, ctxp->ctx2)); + if ((domain->flags & DMAR_DOMAIN_IDMAP) != 0 && (unit->hw_ecap & DMAR_ECAP_PT) != 0) { KASSERT(domain->pgtbl_obj == NULL, ("ctx %p non-null pgtbl_obj", ctx)); - dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); + ctx_root = NULL; } else { ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, DMAR_PGF_NOALLOC); - dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR | - (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | - DMAR_CTX1_P); + } + + if (dmar_is_buswide_ctx(unit, busno)) { + MPASS(!move); + for (i = 0; i <= PCI_BUSMAX; i++) { + ctx_id_entry_init_one(&ctxp[i], domain, ctx_root); + } + } else { + ctx_id_entry_init_one(ctxp, domain, ctx_root); } dmar_flush_ctx_to_ram(unit, ctxp); } @@ -444,6 +465,9 @@ dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid, enable = false; TD_PREP_PINNED_ASSERT; DMAR_LOCK(dmar); + KASSERT(!dmar_is_buswide_ctx(dmar, bus) || (slot == 0 && func == 0), + ("dmar%d pci%d:%d:%d get_ctx for buswide", dmar->unit, bus, + slot, func)); ctx = dmar_find_ctx_locked(dmar, rid); error = 0; if (ctx == NULL) { @@ -492,7 +516,7 @@ dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid, if (LIST_EMPTY(&dmar->domains)) enable = true; LIST_INSERT_HEAD(&dmar->domains, domain, link); - ctx_id_entry_init(ctx, ctxp, false); + ctx_id_entry_init(ctx, ctxp, false, bus); if (dev != NULL) { device_printf(dev, "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d " @@ -597,7 +621,7 @@ dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx) dmar_ctx_unlink(ctx); ctx->domain = domain; dmar_ctx_link(ctx); - ctx_id_entry_init(ctx, ctxp, true); + ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100); dmar_unmap_pgtbl(sf); error = dmar_flush_for_ctx_entry(dmar, true); /* If flush failed, rolling back would not work as well. */ diff --git a/sys/x86/iommu/intel_dmar.h b/sys/x86/iommu/intel_dmar.h index 9a17470ae03..331e939d447 100644 --- a/sys/x86/iommu/intel_dmar.h +++ b/sys/x86/iommu/intel_dmar.h @@ -239,6 +239,15 @@ struct dmar_unit { struct taskqueue *delayed_taskqueue; int dma_enabled; + + /* + * Bitmap of buses for which context must ignore slot:func, + * duplicating the page table pointer into all context table + * entries. This is a client-controlled quirk to support some + * NTBs. + */ + uint32_t buswide_ctxs[(PCI_BUSMAX + 1) / NBBY / sizeof(uint32_t)]; + }; #define DMAR_LOCK(dmar) mtx_lock(&(dmar)->lock) @@ -377,6 +386,9 @@ void dmar_quirks_pre_use(struct dmar_unit *dmar); int dmar_init_irt(struct dmar_unit *unit); void dmar_fini_irt(struct dmar_unit *unit); +void dmar_set_buswide_ctx(struct dmar_unit *unit, u_int busno); +bool dmar_is_buswide_ctx(struct dmar_unit *unit, u_int busno); + #define DMAR_GM_CANWAIT 0x0001 #define DMAR_GM_CANSPLIT 0x0002 diff --git a/sys/x86/iommu/intel_drv.c b/sys/x86/iommu/intel_drv.c index bd475d8b4ca..0805ca96537 100644 --- a/sys/x86/iommu/intel_drv.c +++ b/sys/x86/iommu/intel_drv.c @@ -69,9 +69,9 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include +#include #ifdef DEV_APIC #include "pcib_if.h" @@ -595,6 +595,26 @@ static driver_t dmar_driver = { DRIVER_MODULE(dmar, acpi, dmar_driver, dmar_devclass, 0, 0); MODULE_DEPEND(dmar, acpi, 1, 1, 1); +void +dmar_set_buswide_ctx(struct dmar_unit *unit, u_int busno) +{ + + MPASS(busno <= PCI_BUSMAX); + DMAR_LOCK(unit); + unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] |= + 1 << (busno % (NBBY * sizeof(uint32_t))); + DMAR_UNLOCK(unit); +} + +bool +dmar_is_buswide_ctx(struct dmar_unit *unit, u_int busno) +{ + + MPASS(busno <= PCI_BUSMAX); + return ((unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] & + (1U << (busno % (NBBY * sizeof(uint32_t))))) != 0); +} + static void dmar_print_path(int busno, int depth, const ACPI_DMAR_PCI_PATH *path) { diff --git a/sys/x86/iommu/intel_gas.c b/sys/x86/iommu/intel_gas.c index 4ee6b462b90..b108e9d4562 100644 --- a/sys/x86/iommu/intel_gas.c +++ b/sys/x86/iommu/intel_gas.c @@ -67,6 +67,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include /* diff --git a/sys/x86/iommu/intel_idpgtbl.c b/sys/x86/iommu/intel_idpgtbl.c index 3c8e27e1b94..290d9bf8456 100644 --- a/sys/x86/iommu/intel_idpgtbl.c +++ b/sys/x86/iommu/intel_idpgtbl.c @@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include static int domain_unmap_buf_locked(struct dmar_domain *domain, diff --git a/sys/x86/iommu/intel_intrmap.c b/sys/x86/iommu/intel_intrmap.c index 6ae88243a64..689e5739c25 100644 --- a/sys/x86/iommu/intel_intrmap.c +++ b/sys/x86/iommu/intel_intrmap.c @@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include diff --git a/sys/x86/iommu/intel_qi.c b/sys/x86/iommu/intel_qi.c index 92330047e11..7a55d86d568 100644 --- a/sys/x86/iommu/intel_qi.c +++ b/sys/x86/iommu/intel_qi.c @@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include static bool diff --git a/sys/x86/iommu/intel_quirks.c b/sys/x86/iommu/intel_quirks.c index a0873ea87cc..789b627c7d1 100644 --- a/sys/x86/iommu/intel_quirks.c +++ b/sys/x86/iommu/intel_quirks.c @@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include diff --git a/sys/x86/iommu/intel_utils.c b/sys/x86/iommu/intel_utils.c index 4cff24efe68..dd46c429052 100644 --- a/sys/x86/iommu/intel_utils.c +++ b/sys/x86/iommu/intel_utils.c @@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include u_int diff --git a/sys/x86/x86/busdma_machdep.c b/sys/x86/x86/busdma_machdep.c index a410ff2c045..53da059b05b 100644 --- a/sys/x86/x86/busdma_machdep.c +++ b/sys/x86/x86/busdma_machdep.c @@ -33,6 +33,8 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_acpi.h" + #include #include #include @@ -245,3 +247,10 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat) return (tc->impl->tag_destroy(dmat)); } +#ifndef ACPI_DMAR +bool +bus_dma_dmar_set_buswide(device_t dev) +{ + return (false); +} +#endif