KekenoBSD/src
3
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

arm64: Optimise the repeated TLBI workaround

Browse Source 
It has been reported that the overhead of repeating all TLBI
instructions is too large [1]. The Software Developer Errata Notices
(SDEN) for the relevant Arm CPUs have been updated so a single
"tlbi vale1is, xzr" followed by "dsb ish"  is sufficient to work around
the issues.

Replace the places we repeat TLBI instructions with the new sequence.

[1] https://lore.kernel.org/linux-arm-kernel/20260218164348.2022831-1-mark.rutland@arm.com/

Reviewed by:	kib
Sponsored by:	Arm Ltd
Differential Revision:	https://reviews.freebsd.org/D55646
This commit is contained in:
Andrew Turner
2026-03-05 14:28:01 +00:00
parent 570b8b2575
commit 80b4129bef
1 changed files with 14 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sys/arm64/arm64/pmap.c
+14 -30
View File
@@ -1926,17 +1926,13 @@ pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)
r = TLBI_VA(va);
if (pmap == kernel_pmap) {
pmap_s1_invalidate_kernel(r, final_only);
if (pmap_multiple_tlbi) {
dsb(ish);
pmap_s1_invalidate_kernel(r, final_only);
}
} else {
r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
pmap_s1_invalidate_user(r, final_only);
if (pmap_multiple_tlbi) {
dsb(ish);
pmap_s1_invalidate_user(r, final_only);
}
}
if (pmap_multiple_tlbi) {
dsb(ish);
__asm __volatile("tlbi vale1is, xzr" ::: "memory");
}
dsb(ish);
isb();
@@ -1978,24 +1974,16 @@ pmap_s1_invalidate_strided(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
end = TLBI_VA(eva);
for (r = start; r < end; r += TLBI_VA(stride))
pmap_s1_invalidate_kernel(r, final_only);
if (pmap_multiple_tlbi) {
dsb(ish);
for (r = start; r < end; r += TLBI_VA(stride))
pmap_s1_invalidate_kernel(r, final_only);
}
} else {
start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
start |= TLBI_VA(sva);
end |= TLBI_VA(eva);
for (r = start; r < end; r += TLBI_VA(stride))
pmap_s1_invalidate_user(r, final_only);
if (pmap_multiple_tlbi) {
dsb(ish);
for (r = start; r < end; r += TLBI_VA(stride))
pmap_s1_invalidate_user(r, final_only);
}
}
if (pmap_multiple_tlbi) {
dsb(ish);
__asm __volatile("tlbi vale1is, xzr" ::: "memory");
}
dsb(ish);
isb();
@@ -2036,11 +2024,11 @@ pmap_s1_invalidate_all_kernel(void)
{
dsb(ishst);
__asm __volatile("tlbi vmalle1is");
dsb(ish);
if (pmap_multiple_tlbi) {
__asm __volatile("tlbi vmalle1is");
dsb(ish);
__asm __volatile("tlbi vale1is, xzr" ::: "memory");
}
dsb(ish);
isb();
}
@@ -2058,17 +2046,13 @@ pmap_s1_invalidate_all(pmap_t pmap)
dsb(ishst);
if (pmap == kernel_pmap) {
__asm __volatile("tlbi vmalle1is");
if (pmap_multiple_tlbi) {
dsb(ish);
__asm __volatile("tlbi vmalle1is");
}
} else {
r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
__asm __volatile("tlbi aside1is, %0" : : "r" (r));
if (pmap_multiple_tlbi) {
dsb(ish);
__asm __volatile("tlbi aside1is, %0" : : "r" (r));
}
}
if (pmap_multiple_tlbi) {
dsb(ish);
__asm __volatile("tlbi vale1is, xzr" ::: "memory");
}
dsb(ish);
isb();