arm64: VM/PMAP changes for CCA guest support

When in a realm:

- Mappings with mode VM_MEMATTR_DEVICE and VM_MEMATTR_DEVICE_NP are
  unprotected
- Imported busdma buffers in protected memory are always bounced
- If EARLY_PRINTK is in use, the UART physical address must be in the
  unprotected address space

Reviewed by:	andrew
Sponsored by:	Arm Ltd
Differential Revision:	https://reviews.freebsd.org/D56599

sys/arm64/arm64/busdma_bounce.c

@@ -36,6 +36,7 @@
 #include <sys/domainset.h>
 #include <sys/malloc.h>
 #include <sys/bus.h>
+#include <sys/busdma_bufalloc.h>
 #include <sys/interrupt.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
@@ -56,6 +57,7 @@
 #include <machine/atomic.h>
 #include <machine/bus.h>
 #include <machine/md_var.h>
+#include <machine/rsi.h>
 #include <arm64/include/bus_dma_impl.h>
 
 #define MAX_BPAGES 4096
@@ -120,6 +122,8 @@ static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
     vm_paddr_t buf, bus_size_t buflen, int flags);
 
+static busdma_bufalloc_t nonsecure_allocator;
+
 static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata");
 
 #define	dmat_alignment(dmat)	((dmat)->common.alignment)
@@ -215,6 +219,10 @@ might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr,
 	if (map && (map->flags & DMAMAP_FROM_DMAMEM) != 0)
 		return (false);
 
+	/* Bounce if accessing secure memory */
+	if (in_realm() && !(paddr & prot_ns_shared_pa))
+		return (true);
+
 	if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0)
 		return (true);
 
@@ -239,6 +247,10 @@ must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr,
 	    addr_needs_bounce(dmat, paddr))
 		return (true);
 
+	/* Bounce if accessing secure memory */
+	if (in_realm() && !(paddr & prot_ns_shared_pa))
+		return (true);
+
 	return (false);
 }
 
@@ -492,6 +504,7 @@ static int
 bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
     bus_dmamap_t *mapp)
 {
+	struct busdma_bufzone *bufzone;
 	vm_memattr_t attr;
 	int mflags;
 
@@ -524,6 +537,9 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 	else
 		attr = VM_MEMATTR_DEFAULT;
 
+	if (in_realm())
+		mflags |= M_UNPROTECTED;
+
 	/*
 	 * Create the map, but don't set the could bounce flag as
 	 * this allocation should never bounce;
@@ -567,13 +583,16 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 	 *
 	 * In the meantime warn the user if malloc gets it wrong.
 	 */
-	if (dmat->alloc_size <= PAGE_SIZE &&
+
+	bufzone = busdma_bufalloc_findzone(nonsecure_allocator,
+	    dmat->alloc_size);
+
+	if (bufzone &&
+	    dmat->alloc_size <= PAGE_SIZE &&
 	    dmat->alloc_alignment <= PAGE_SIZE &&
 	    dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
 	    attr == VM_MEMATTR_DEFAULT) {
-		*vaddr = malloc_domainset_aligned(dmat->alloc_size,
-		    dmat->alloc_alignment, M_DEVBUF,
-		    DOMAINSET_PREF(dmat->common.domain), mflags);
+		*vaddr = uma_zalloc(bufzone->umazone, mflags);
 	} else if (dmat->common.nsegments >=
 	    howmany(dmat->alloc_size, MIN(dmat->common.maxsegsz, PAGE_SIZE)) &&
 	    dmat->alloc_alignment <= PAGE_SIZE &&
@@ -1148,3 +1167,44 @@ struct bus_dma_impl bus_dma_bounce_impl = {
 	.load_kmsan = bounce_bus_dmamap_load_kmsan,
 #endif
 };
+
+static void *
+nonsecure_alloc(uma_zone_t zone, vm_size_t size, int domain, uint8_t *pflag,
+    int wait)
+{
+	void *p;	/* Returned page */
+
+	*pflag = UMA_SLAB_KERNEL;
+	p = kmem_malloc_domainset(DOMAINSET_FIXED(domain), size,
+	    wait | M_UNPROTECTED);
+
+	return (p);
+}
+
+static void
+nonsecure_free(void *mem, vm_size_t size, uint8_t flags)
+{
+	KASSERT((flags & UMA_SLAB_KERNEL) != 0,
+	    ("UMA: page_free used with invalid flags %x", flags));
+
+	kmem_free(mem, size);
+}
+
+static void
+busdma_bounce_init(void *dummy)
+{
+	if (in_realm())
+		nonsecure_allocator = busdma_bufalloc_create("nonsecure",
+		    dcache_line_size,		/* minimum_alignment */
+		    nonsecure_alloc,		/* uma_alloc func */
+		    nonsecure_free,		/* uma_free func */
+		    UMA_ZONE_NOTOUCH);		/* uma_zcreate_flags */
+	else
+		nonsecure_allocator = busdma_bufalloc_create("nonsecure",
+		    dcache_line_size,		/* minimum_alignment */
+		    NULL,			/* uma_alloc func */
+		    NULL,			/* uma_free func */
+		    UMA_ZONE_NOTOUCH);		/* uma_zcreate_flags */
+}
+
+SYSINIT(busdma, SI_SUB_KMEM + 1, SI_ORDER_FIRST, busdma_bounce_init, NULL);

sys/arm64/arm64/efirt_machdep.c

@@ -46,6 +46,7 @@
 #include <sys/vmmeter.h>
 
 #include <machine/pte.h>
+#include <machine/rsi.h>
 #include <machine/vmparam.h>
 
 #include <vm/vm.h>
@@ -171,6 +172,7 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz)
 	vm_page_t efi_l0_page;
 	uint64_t idx;
 	int i, mode;
+	uint64_t pa;
 
 	obj_1t1_pt = vm_pager_allocate(OBJT_PHYS, NULL, L0_ENTRIES +
 	    L0_ENTRIES * Ln_ENTRIES + L0_ENTRIES * Ln_ENTRIES * Ln_ENTRIES +
@@ -231,7 +233,13 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz)
 		for (va = p->md_phys, idx = 0; idx < p->md_pages;
 		    idx += (PAGE_SIZE / EFI_PAGE_SIZE), va += PAGE_SIZE) {
 			l3 = efi_1t1_l3(va);
-			*l3 = va | l3_attr;
+
+			if (mode == VM_MEMATTR_DEVICE && in_realm())
+				pa = va | prot_ns_shared_pa;
+			else
+				pa = va;
+
+			*l3 = PHYS_TO_PTE(pa) | l3_attr;
 		}
 		VM_OBJECT_WUNLOCK(obj_1t1_pt);
 	}

sys/arm64/arm64/pmap.c

@@ -152,6 +152,7 @@
 #include <machine/machdep.h>
 #include <machine/md_var.h>
 #include <machine/pcb.h>
+#include <machine/rsi.h>
 
 #ifdef NUMA
 #define	PMAP_MEMDOM	MAXMEMDOM
@@ -539,6 +540,9 @@ static int pmap_bti_copy(pmap_t dst_pmap, pmap_t src_pmap);
 static void pmap_bti_deassign_all(pmap_t pmap);
 static void pagezero(void *);
 
+static void pmap_set_protected(pt_entry_t old_l3);
+static void pmap_set_unprotected(pt_entry_t new_l3);
+
 /*
  * These load the old table data and store the new value.
  * They need to be atomic as the System MMU may write to the table at
@@ -2381,6 +2385,11 @@ pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode)
 	KASSERT((size & PAGE_MASK) == 0,
 	    ("pmap_kenter: Mapping is not page-sized"));
 
+	/* CCA - Map devices as nonsecure */
+	if (in_realm() && (mode == VM_MEMATTR_DEVICE ||
+	    mode == VM_MEMATTR_DEVICE_NP))
+		pa |= prot_ns_shared_pa;
+
 	attr = ATTR_AF | pmap_sh_attr | ATTR_S1_AP(ATTR_S1_AP_RW) |
 	    ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(mode);
 	old_l3e = 0;
@@ -4224,6 +4233,9 @@ pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva,
 		if ((old_l3 & ATTR_SW_WIRED) != 0)
 			pmap->pm_stats.wired_count--;
 		pmap_resident_count_dec(pmap, 1);
+		/* Below will only be true in a realm environment. */
+		if (PTE_TO_PHYS(old_l3) & prot_ns_shared_pa)
+			pmap_set_protected(old_l3);
 		if ((old_l3 & ATTR_SW_MANAGED) != 0) {
 			m = PTE_TO_VM_PAGE(old_l3);
 			if (pmap_pte_dirty(pmap, old_l3))
@@ -5376,6 +5388,28 @@ pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t pte, int flags,
 	return (KERN_SUCCESS);
 }
 
+static void
+pmap_set_unprotected(pt_entry_t new_l3)
+{
+	vm_paddr_t pa;
+
+	pa = PTE_TO_PHYS(new_l3) & ~prot_ns_shared_pa;
+
+	rsi_set_addr_range_state(pa, pa + L3_SIZE, RSI_RIPAS_EMPTY,
+	    RSI_CHANGE_DESTROYED, NULL);
+}
+
+static void
+pmap_set_protected(pt_entry_t old_l3)
+{
+	vm_paddr_t pa;
+
+	pa = PTE_TO_PHYS(old_l3) & ~prot_ns_shared_pa;
+
+	rsi_set_addr_range_state(pa, pa + L3_SIZE, RSI_RIPAS_RAM,
+	    RSI_CHANGE_DESTROYED, NULL);
+}
+
 /*
  *	Insert the given physical page (p) at
  *	the specified virtual address (v) in the
@@ -5409,6 +5443,8 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 	if ((m->oflags & VPO_UNMANAGED) == 0)
 		VM_PAGE_OBJECT_BUSY_ASSERT(m);
 	pa = VM_PAGE_TO_PHYS(m);
+	if (in_realm() && (flags & PMAP_ENTER_UNPROTECTED) != 0)
+		pa |= prot_ns_shared_pa;
 	new_l3 = (pt_entry_t)(PHYS_TO_PTE(pa) | ATTR_AF | pmap_sh_attr |
 	    L3_PAGE);
 	new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr);
@@ -5728,6 +5764,10 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 #endif
 
 	rv = KERN_SUCCESS;
+
+	if (in_realm() && (flags & PMAP_ENTER_UNPROTECTED) != 0)
+		pmap_set_unprotected(new_l3);
+
 out:
 	if (lock != NULL)
 		rw_wunlock(lock);

sys/conf/files.arm64

@@ -9,6 +9,7 @@ kern/subr_devmap.c				standard
 kern/subr_efi_map.c				standard
 kern/subr_intr.c				optional intrng
 kern/subr_physmem.c				standard
+kern/subr_busdma_bufalloc.c			standard
 libkern/strlen.c		standard
 libkern/arm64/crc32c_armv8.S			standard
 

sys/vm/vm_page.c

@@ -110,6 +110,10 @@
 #include <vm/uma_int.h>
 
 #include <machine/md_var.h>
+#if defined(__aarch64__)
+#include <machine/pmap.h>
+#include <machine/rsi.h>
+#endif
 
 struct vm_domain vm_dom[MAXMEMDOM];
 
@@ -1290,6 +1294,10 @@ PHYS_TO_VM_PAGE(vm_paddr_t pa)
 	vm_page_t m;
 
 #ifdef VM_PHYSSEG_SPARSE
+#if defined(__aarch64__)
+	if (in_realm())
+		pa &= ~prot_ns_shared_pa; /* Mask off secure bit */
+#endif
 	m = vm_phys_paddr_to_vm_page(pa);
 	if (m == NULL)
 		m = vm_phys_fictitious_to_vm_page(pa);