arm64: Support TBI in userspace

To allow for Hardware-assisted AddressSanitizer (HWASAN) and future
work to enable MTE we need to enable TBI in userspace. As address space
that previously would have faulted will now not it could be considered
an ABI change so only enable for processes with a late enough revision.

Relnotes:	yes
Sponsored by:	Arm Ltd
Differential Revision:	https://reviews.freebsd.org/D51637

sys/arm64/arm64/efirt_machdep.c

@@ -241,6 +241,7 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz)
 int
 efi_arch_enter(void)
 {
+	uint64_t tcr;
 
 	CRITICAL_ASSERT(curthread);
 	curthread->td_md.md_efirt_dis_pf = vm_fault_disable_pagefaults();
@@ -249,7 +250,17 @@ efi_arch_enter(void)
 	 * Temporarily switch to EFI's page table.  However, we leave curpmap
 	 * unchanged in order to prevent its ASID from being reclaimed before
 	 * we switch back to its page table in efi_arch_leave().
+	 *
+	 * UEFI sdoesn't care about TBI, so enable it. It's more likely
+	 * userspace will have TBI on as it's only disabled for backwards
+	 * compatibility.
 	 */
+	tcr = READ_SPECIALREG(tcr_el1);
+	if ((tcr & MD_TCR_FIELDS) != TCR_TBI0) {
+		tcr &= ~MD_TCR_FIELDS;
+		tcr |= TCR_TBI0;
+		WRITE_SPECIALREG(tcr_el1, tcr);
+	}
 	set_ttbr0(efi_ttbr0);
 	if (PCPU_GET(bcast_tlbi_workaround) != 0)
 		invalidate_local_icache();
@@ -260,6 +271,7 @@ efi_arch_enter(void)
 void
 efi_arch_leave(void)
 {
+	uint64_t proc_tcr, tcr;
 
 	/*
 	 * Restore the pcpu pointer. Some UEFI implementations trash it and
@@ -271,6 +283,13 @@ efi_arch_leave(void)
 	__asm __volatile(
 	    "mrs x18, tpidr_el1	\n"
 	);
+	proc_tcr = curthread->td_proc->p_md.md_tcr;
+	tcr = READ_SPECIALREG(tcr_el1);
+	if ((tcr & MD_TCR_FIELDS) != proc_tcr) {
+		tcr &= ~MD_TCR_FIELDS;
+		tcr |= proc_tcr;
+		WRITE_SPECIALREG(tcr_el1, tcr);
+	}
 	set_ttbr0(pmap_to_ttbr0(PCPU_GET(curpmap)));
 	if (PCPU_GET(bcast_tlbi_workaround) != 0)
 		invalidate_local_icache();

sys/arm64/arm64/elf_machdep.c

@@ -65,7 +65,13 @@ u_long __read_frequently linux_elf_hwcap2;
 u_long __read_frequently linux_elf_hwcap3;
 u_long __read_frequently linux_elf_hwcap4;
 
-struct arm64_addr_mask elf64_addr_mask;
+struct arm64_addr_mask elf64_addr_mask = {
+    .code = TBI_ADDR_MASK,
+    .data = TBI_ADDR_MASK,
+};
+#ifdef COMPAT_FREEBSD14
+struct arm64_addr_mask elf64_addr_mask_14;
+#endif
 
 static void arm64_exec_protect(struct image_params *, int);
 
@@ -136,7 +142,14 @@ get_arm64_addr_mask(struct regset *rs, struct thread *td, void *buf,
 	if (buf != NULL) {
 		KASSERT(*sizep == sizeof(elf64_addr_mask),
 		    ("%s: invalid size", __func__));
-		memcpy(buf, &elf64_addr_mask, sizeof(elf64_addr_mask));
+#ifdef COMPAT_FREEBSD14
+		/* running an old binary use the old address mask */
+		if (td->td_proc->p_osrel < TBI_VERSION)
+			memcpy(buf, &elf64_addr_mask_14,
+			    sizeof(elf64_addr_mask_14));
+		else
+#endif
+			memcpy(buf, &elf64_addr_mask, sizeof(elf64_addr_mask));
 	}
 	*sizep = sizeof(elf64_addr_mask);
 

sys/arm64/arm64/exec_machdep.c

@@ -51,6 +51,7 @@
 #include <vm/vm_map.h>
 
 #include <machine/armreg.h>
+#include <machine/elf.h>
 #include <machine/kdb.h>
 #include <machine/md_var.h>
 #include <machine/pcb.h>
@@ -411,6 +412,7 @@ exec_setregs(struct thread *td, struct image_params *imgp, uintptr_t stack)
 {
 	struct trapframe *tf = td->td_frame;
 	struct pcb *pcb = td->td_pcb;
+	uint64_t new_tcr, tcr;
 
 	memset(tf, 0, sizeof(struct trapframe));
 
@@ -433,6 +435,35 @@ exec_setregs(struct thread *td, struct image_params *imgp, uintptr_t stack)
 	 */
 	bzero(&pcb->pcb_dbg_regs, sizeof(pcb->pcb_dbg_regs));
 
+	/* If the process is new enough enable TBI */
+	if (td->td_proc->p_osrel >= TBI_VERSION)
+		new_tcr = TCR_TBI0;
+	else
+		new_tcr = 0;
+	td->td_proc->p_md.md_tcr = new_tcr;
+
+	/* TODO: should create a pmap function for this... */
+	tcr = READ_SPECIALREG(tcr_el1);
+	if ((tcr & MD_TCR_FIELDS) != new_tcr) {
+		uint64_t asid;
+
+		tcr &= ~MD_TCR_FIELDS;
+		tcr |= new_tcr;
+		WRITE_SPECIALREG(tcr_el1, tcr);
+		isb();
+
+		/*
+		 * TCR_EL1.TBI0 is permitted to be cached in the TLB, so
+		 * we need to perform a TLB invalidation.
+		 */
+		asid = READ_SPECIALREG(ttbr0_el1) & TTBR_ASID_MASK;
+		__asm __volatile(
+		    "tlbi aside1is, %0		\n"
+		    "dsb ish			\n"
+		    "isb			\n"
+		    : : "r" (asid));
+	}
+
 	/* Generate new pointer authentication keys */
 	ptrauth_exec(td);
 }

sys/arm64/arm64/genassym.c

@@ -64,6 +64,7 @@ ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault));
 ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags));
 
 ASSYM(P_PID, offsetof(struct proc, p_pid));
+ASSYM(P_MD_TCR, offsetof(struct proc, p_md.md_tcr));
 
 ASSYM(SF_UC, offsetof(struct sigframe, sf_uc));
 

sys/arm64/arm64/pmap.c

@@ -469,7 +469,7 @@ static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
 		    vm_offset_t va);
 
 static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);
-static bool pmap_activate_int(pmap_t pmap);
+static bool pmap_activate_int(struct thread *td, pmap_t pmap);
 static void pmap_alloc_asid(pmap_t pmap);
 static int pmap_change_props_locked(vm_offset_t va, vm_size_t size,
     vm_prot_t prot, int mode, bool skip_unmapped);
@@ -9113,7 +9113,7 @@ pmap_init_cnp(void *dummy __unused)
 SYSINIT(pmap_init_cnp, SI_SUB_SMP, SI_ORDER_ANY, pmap_init_cnp, NULL);
 
 static bool
-pmap_activate_int(pmap_t pmap)
+pmap_activate_int(struct thread *td, pmap_t pmap)
 {
 	struct asid_set *set;
 	int epoch;
@@ -9152,6 +9152,15 @@ pmap_activate_int(pmap_t pmap)
 		pmap_alloc_asid(pmap);
 
 	if (pmap->pm_stage == PM_STAGE1) {
+		uint64_t new_tcr, tcr;
+
+		new_tcr = td->td_proc->p_md.md_tcr;
+		tcr = READ_SPECIALREG(tcr_el1);
+		if ((tcr & MD_TCR_FIELDS) != new_tcr) {
+			tcr &= ~MD_TCR_FIELDS;
+			tcr |= new_tcr;
+			WRITE_SPECIALREG(tcr_el1, tcr);
+		}
 		set_ttbr0(pmap_to_ttbr0(pmap));
 		if (PCPU_GET(bcast_tlbi_workaround) != 0)
 			invalidate_local_icache();
@@ -9165,7 +9174,7 @@ pmap_activate_vm(pmap_t pmap)
 
 	PMAP_ASSERT_STAGE2(pmap);
 
-	(void)pmap_activate_int(pmap);
+	(void)pmap_activate_int(NULL, pmap);
 }
 
 void
@@ -9176,7 +9185,7 @@ pmap_activate(struct thread *td)
 	pmap = vmspace_pmap(td->td_proc->p_vmspace);
 	PMAP_ASSERT_STAGE1(pmap);
 	critical_enter();
-	(void)pmap_activate_int(pmap);
+	(void)pmap_activate_int(td, pmap);
 	critical_exit();
 }
 
@@ -9202,7 +9211,7 @@ pmap_switch(struct thread *new)
 	 * to a user process.
 	 */
 
-	if (pmap_activate_int(vmspace_pmap(new->td_proc->p_vmspace))) {
+	if (pmap_activate_int(new, vmspace_pmap(new->td_proc->p_vmspace))) {
 		/*
 		 * Stop userspace from training the branch predictor against
 		 * other processes. This will call into a CPU specific

sys/arm64/arm64/ptrauth.c

@@ -149,6 +149,10 @@ ptrauth_enable(const struct cpu_feat *feat __unused,
 	enable_ptrauth = true;
 	elf64_addr_mask.code |= PAC_ADDR_MASK;
 	elf64_addr_mask.data |= PAC_ADDR_MASK;
+#ifdef COMPAT_FREEBSD14
+	elf64_addr_mask_14.code |= PAC_ADDR_MASK_14;
+	elf64_addr_mask_14.data |= PAC_ADDR_MASK_14;
+#endif
 }
 
 

sys/arm64/arm64/swtch.S

@@ -37,6 +37,8 @@
 
 #include <machine/asm.h>
 #include <machine/armreg.h>
+#include <machine/proc.h>
+
 .macro clear_step_flag pcbflags, tmp
 	tbz	\pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
 	mrs	\tmp, mdscr_el1
@@ -239,6 +241,16 @@ ENTRY(fork_trampoline)
 	msr	daifset, #(DAIF_D | DAIF_INTR)
 
 	ldr	x0, [x18, #PC_CURTHREAD]
+
+	/* Set the per-process tcr_el1 fields */
+	ldr	x10, [x0, #TD_PROC]
+	ldr	x10, [x10, #P_MD_TCR]
+	mrs	x11, tcr_el1
+	and	x11, x11, #(~MD_TCR_FIELDS)
+	orr	x11, x11, x10
+	msr	tcr_el1, x11
+	/* No isb as the eret below is the context-synchronising event */
+
 	bl	ptrauth_enter_el0
 
 	/* Restore sp, lr, elr, and spsr */

sys/arm64/arm64/vm_machdep.c

@@ -120,6 +120,9 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 	td2->td_md.md_spinlock_count = 1;
 	td2->td_md.md_saved_daif = PSR_DAIF_DEFAULT;
 
+	/* Copy the TCR_EL1 value */
+	td2->td_proc->p_md.md_tcr = td1->td_proc->p_md.md_tcr;
+
 #if defined(PERTHREAD_SSP)
 	/* Set the new canary */
 	arc4random_buf(&td2->td_md.md_canary, sizeof(td2->td_md.md_canary));

sys/arm64/include/cpu.h

@@ -226,6 +226,9 @@ extern uint64_t __cpu_affinity[];
 
 struct arm64_addr_mask;
 extern struct arm64_addr_mask elf64_addr_mask;
+#ifdef COMPAT_FREEBSD14
+extern struct arm64_addr_mask elf64_addr_mask_14;
+#endif
 
 typedef void (*cpu_reset_hook_t)(void);
 extern cpu_reset_hook_t cpu_reset_hook;

sys/arm64/include/elf.h

@@ -93,6 +93,9 @@ __ElfType(Auxinfo);
 #define	ET_DYN_LOAD_ADDR 0x100000
 #endif
 
+/* First __FreeBSD_version that supports Top Byte Ignore (TBI) */
+#define	TBI_VERSION	1500058
+
 /* HWCAP */
 #define	HWCAP_FP		(1 << 0)
 #define	HWCAP_ASIMD		(1 << 1)

sys/arm64/include/proc.h

@@ -35,6 +35,7 @@
 #ifndef	_MACHINE_PROC_H_
 #define	_MACHINE_PROC_H_
 
+#ifndef LOCORE
 struct ptrauth_key {
 	uint64_t pa_key_lo;
 	uint64_t pa_key_hi;
@@ -73,8 +74,12 @@ struct mdthread {
 };
 
 struct mdproc {
-	long	md_dummy;
+	uint64_t md_tcr;		/* TCR_EL1 fields to update */
 };
+#endif /* !LOCORE */
+
+/* Fields that can be set in md_tcr */
+#define	MD_TCR_FIELDS			TCR_TBI0
 
 #define	KINFO_PROC_SIZE	1088
 #define	KINFO_PROC32_SIZE 816

sys/arm64/include/vmparam.h

@@ -209,7 +209,8 @@
 #define	KMSAN_ORIG_MAX_ADDRESS	(0xffff028000000000UL)
 
 /* The address bits that hold a pointer authentication code */
-#define	PAC_ADDR_MASK		(0xff7f000000000000UL)
+#define	PAC_ADDR_MASK		(0x007f000000000000UL)
+#define	PAC_ADDR_MASK_14	(0xff7f000000000000UL)
 
 /* The top-byte ignore address bits */
 #define	TBI_ADDR_MASK		0xff00000000000000UL

sys/sys/param.h

@@ -74,7 +74,7 @@
  * cannot include sys/param.h and should only be updated here.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1500057
+#define __FreeBSD_version 1500058
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,

tests/sys/kern/ptrace_test.c

@@ -3239,7 +3239,7 @@ ATF_TC_BODY(ptrace__PT_REGSET, tc)
 	ATF_REQUIRE(ptrace(PT_GETREGSET, wpid, (caddr_t)&vec,
 	    NT_ARM_ADDR_MASK) != -1);
 	REQUIRE_EQ(addr_mask.code, addr_mask.data);
-	ATF_REQUIRE(addr_mask.code == 0 ||
+	ATF_REQUIRE(addr_mask.code == 0xff00000000000000ul ||
 	    addr_mask.code == 0xff7f000000000000UL);
 #endif