arm64: Check for MTE tag failures on kernel entry

When entering the kernel from userspace we need to check for MTE tag
failures when using asynchronous MTE. This is done by checking if either
tag fault check types that have asynchronous checks are enabled, and
if so check the register the result is stored. It then sets a flag the
kernel can later use to raise a signal.

Sponsored by:	Arm Ltd
Differential Revision:	https://reviews.freebsd.org/D55952

sys/arm64/arm64/exception.S

@@ -92,10 +92,34 @@
 	blr	x1
 1:
 
-	ldr	x0, [x18, #PC_CURTHREAD]
+	ldr	x19, [x18, #PC_CURTHREAD]
+
+	ldr	x1, [x19, #TD_MD_SCTLR]
+	/*
+	 * If the upper bit in SCTLR_EL1.TCF0 is set we are either in async
+	 * or asym modes. Either of which could set TFSRE0_EL1.
+	 */
+	tbz	x1, #(SCTLR_TCF0_SHIFT + 1), 2f
+	/* Check for a tag fault */
+	mrs	x1, TFSRE0_EL1_REG
+	tbz	x1, #TFSRE0_TF0_SHIFT, 2f
+
+	/*
+	 * A fault has happened, set MD_FLAG_MTE_ASYNC_FAULT. As FEAT_LSE
+	 * is a required feature where FEAT_MTE_ASYNC could be implemented
+	 * we can depend on it being present to set the flag.
+	 */
+	ldr	w1, =MD_FLAG_MTE_ASYNC_FAULT
+	add	x2, x19, #TD_MD_FLAGS
+.arch_extension lse
+	stset	w1, [x2]
+.arch_extension nolse
+
+2:
+	mov	x0, x19
 	bl	ptrauth_exit_el0
 
-	ldr	x0, [x18, #(PC_CURTHREAD)]
+	mov	x0, x19
 	bl	dbg_monitor_enter
 
 	/* Unmask debug and SError exceptions */
@@ -118,6 +142,14 @@
 	msr	daifset, #(DAIF_ALL)
 .if \el == 0
 	ldr	x0, [x18, #PC_CURTHREAD]
+
+	ldr	x1, [x0, #TD_MD_SCTLR]
+	/* See above for why we check this field */
+	tbz	x1, #(SCTLR_TCF0_SHIFT + 1), 1f
+	dsb	ish
+	msr	TFSRE0_EL1_REG, xzr
+1:
+
 	mov	x1, sp
 	bl	dbg_monitor_exit
 

sys/arm64/arm64/genassym.c

@@ -74,6 +74,9 @@ ASSYM(TD_FRAME, offsetof(struct thread, td_frame));
 ASSYM(TD_LOCK, offsetof(struct thread, td_lock));
 ASSYM(TD_MD_CANARY, offsetof(struct thread, td_md.md_canary));
 ASSYM(TD_MD_EFIRT_TMP, offsetof(struct thread, td_md.md_efirt_tmp));
+ASSYM(TD_MD_FLAGS, offsetof(struct thread, td_md.md_flags));
+ASSYM(MD_FLAG_MTE_ASYNC_FAULT, MD_FLAG_MTE_ASYNC_FAULT);
+ASSYM(TD_MD_SCTLR, offsetof(struct thread, td_md.md_sctlr));
 
 ASSYM(TF_SIZE, sizeof(struct trapframe));
 ASSYM(TF_SP, offsetof(struct trapframe, tf_sp));

sys/arm64/include/proc.h

@@ -69,7 +69,9 @@ struct mdthread {
 	uint64_t md_efirt_tmp;
 	int md_efirt_dis_pf;
 
-	int md_reserved0;
+	u_int md_flags;
+#define	MD_FLAG_MTE_ASYNC_FAULT_SHIFT	0
+#define	MD_FLAG_MTE_ASYNC_FAULT		(1u << 0)
 	uint64_t md_sctlr;
 	uint64_t md_gcr;		/* FEAT_MTE: Tag Control Register */
 };