From 2499a5ccef046dcacd780a464e91ffbb6d6fdde9 Mon Sep 17 00:00:00 2001 From: Konstantin Belousov Date: Thu, 3 Jul 2014 11:06:27 +0000 Subject: [PATCH] Micro-manage clang to get the expected inlining for cpu_search(). Mark cpu_search_lowest/cpu_search_highest/cpu_search_both as noinline, while cpu_search() gets always_inline. With the attributes set, cpu_search() is inlined in wrappers, and if()s with constant conditionals are optimized. On some tests on many-core machine, the hwpmc reported samples for cpu_search*() are reduced from 25% total to 9%. Submitted by: "Rang, Anton" MFC after: 1 week --- sys/kern/sched_ule.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c index cc737e3b632..12743b2a944 100644 --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -622,12 +622,14 @@ struct cpu_search { for ((cpu) = 0; (cpu) <= mp_maxid; (cpu)++) \ if (CPU_ISSET(cpu, &mask)) -static __inline int cpu_search(const struct cpu_group *cg, struct cpu_search *low, - struct cpu_search *high, const int match); -int cpu_search_lowest(const struct cpu_group *cg, struct cpu_search *low); -int cpu_search_highest(const struct cpu_group *cg, struct cpu_search *high); -int cpu_search_both(const struct cpu_group *cg, struct cpu_search *low, +static __always_inline int cpu_search(const struct cpu_group *cg, + struct cpu_search *low, struct cpu_search *high, const int match); +int __noinline cpu_search_lowest(const struct cpu_group *cg, + struct cpu_search *low); +int __noinline cpu_search_highest(const struct cpu_group *cg, struct cpu_search *high); +int __noinline cpu_search_both(const struct cpu_group *cg, + struct cpu_search *low, struct cpu_search *high); /* * Search the tree of cpu_groups for the lowest or highest loaded cpu @@ -640,7 +642,7 @@ int cpu_search_both(const struct cpu_group *cg, struct cpu_search *low, * match argument. It is reduced to the minimum set for each case. It is * also recursive to the depth of the tree. */ -static __inline int +static __always_inline int cpu_search(const struct cpu_group *cg, struct cpu_search *low, struct cpu_search *high, const int match) {