From f7cc0eae7ec570f27458417c044bb9a447833685 Mon Sep 17 00:00:00 2001
From: Mitchell Horne <mhorne@FreeBSD.org>
Date: Fri, 11 Dec 2020 20:01:45 +0000
Subject: [PATCH] riscv: small counter(9) improvements

Prefer atomics to critical section. This reduces the cost of the
increment operation and removes the possibility of it being interrupted
by counter_u64_zero().

Use CPU_FOREACH() macro to skip absent CPUs.

Replace hand-rolled address calculation with zpcpu_get().

Reviewed by:	markj
Differential Revision:	https://reviews.freebsd.org/D27536
---
 sys/riscv/include/counter.h | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/sys/riscv/include/counter.h b/sys/riscv/include/counter.h
index 038c49f55f8..46a60f40026 100644
--- a/sys/riscv/include/counter.h
+++ b/sys/riscv/include/counter.h
@@ -30,14 +30,12 @@
 #define	_MACHINE_COUNTER_H_
 
 #include <sys/pcpu.h>
-#ifdef INVARIANTS
-#include <sys/proc.h>
-#endif
+#include <machine/atomic.h>
 
 #define	EARLY_COUNTER	&__pcpu[0].pc_early_dummy_counter
 
-#define	counter_enter()	critical_enter()
-#define	counter_exit()	critical_exit()
+#define	counter_enter()	do {} while (0)
+#define	counter_exit()	do {} while (0)
 
 #ifdef IN_SUBR_COUNTER_C
 static inline uint64_t
@@ -54,19 +52,17 @@ counter_u64_fetch_inline(uint64_t *p)
 	int i;
 
 	r = 0;
-	for (i = 0; i < mp_ncpus; i++)
-		r += counter_u64_read_one((uint64_t *)p, i);
+	CPU_FOREACH(i)
+		r += counter_u64_read_one(p, i);
 
 	return (r);
 }
 
-/* XXXKIB might interrupt increment */
 static void
 counter_u64_zero_one_cpu(void *arg)
 {
 
-	*((uint64_t *)((char *)arg + UMA_PCPU_ALLOC_SIZE *
-	    PCPU_GET(cpuid))) = 0;
+	*(zpcpu_get((counter_u64_t *)arg)) = 0;
 }
 
 static inline void
@@ -78,18 +74,13 @@ counter_u64_zero_inline(counter_u64_t c)
 }
 #endif
 
-#define	counter_u64_add_protected(c, inc)	do {	\
-	CRITICAL_ASSERT(curthread);			\
-	*(uint64_t *)zpcpu_get(c) += (inc);		\
-} while (0)
+#define	counter_u64_add_protected(c, inc)	counter_u64_add(c, inc)
 
 static inline void
 counter_u64_add(counter_u64_t c, int64_t inc)
 {
 
-	counter_enter();
-	counter_u64_add_protected(c, inc);
-	counter_exit();
+	atomic_add_64((uint64_t *)zpcpu_get(c), inc);
 }
 
 #endif	/* ! _MACHINE_COUNTER_H_ */