hwpmc: Add IBS capability control policy

Reject unsupported AMD IBS and PMU control bits before programming the
MSRs.

Initialize IBS fetch/op allow masks from CPUID feature bits and validate
user-provided IBS control values against those masks. Keep the
load-latency filter dependency on L3MissOnly, but avoid decoding fields
that are already constrained by the mask.

Apply the same reserved-bit policy to the AMD PMU raw-config path by
checking core, L3, and data fabric configs against subclass-specific
masks.

Fix the IBS CPUID feature bit definitions used by the policy.

Reviewed by:	mhorne, Ali Mashtizadeh <ali@mashtizadeh.com>
Sponsored by:	AMD
Signed-off-by:	Andre Silva <andasilv@amd.com>
Pull Request:	https://github.com/freebsd/freebsd-src/pull/2140

sys/dev/hwpmc/hwpmc_amd.c

@@ -178,6 +178,45 @@ struct amd_cpu {
 };
 static struct amd_cpu **amd_pcpu;
 
+/* Populated by amd_init_policy(); PRECISERETIRE is OR-ed in per-allocation. */
+static uint64_t amd_core_allowed_mask;
+static uint64_t amd_l3_allowed_mask;
+static uint64_t amd_df_allowed_mask;
+
+static void
+amd_init_policy(void)
+{
+	int family;
+
+	family = CPUID_TO_FAMILY(cpu_id);
+
+	amd_core_allowed_mask = AMD_VALID_BITS;
+
+	amd_l3_allowed_mask = (family <= 0x17) ?
+	    AMD_PMC_L3_FAMILY17_MASK : AMD_PMC_L3_FAMILY19_MASK;
+
+	amd_df_allowed_mask = (family <= 0x19) ?
+	    AMD_PMC_DF_FAMILY17_MASK : AMD_PMC_DF_FAMILY1A_MASK;
+}
+
+static uint64_t
+amd_config_mask(enum sub_class subclass, uint64_t caps)
+{
+
+	switch (subclass) {
+	case PMC_AMD_SUB_CLASS_CORE:
+		return (amd_core_allowed_mask |
+		    (((caps & PMC_CAP_PRECISE) != 0) ?
+		    AMD_PMC_PRECISERETIRE : 0));
+	case PMC_AMD_SUB_CLASS_L3_CACHE:
+		return (amd_l3_allowed_mask);
+	case PMC_AMD_SUB_CLASS_DATA_FABRIC:
+		return (amd_df_allowed_mask);
+	default:
+		return (0);
+	}
+}
+
 /*
  * Read a PMC value from the MSR.
  */
@@ -358,9 +397,13 @@ amd_allocate_pmc(int cpu __unused, int ri, struct pmc *pm,
 		return (EINVAL);
 
 	if (strlen(pmc_cpuid) != 0) {
-		pm->pm_md.pm_amd.pm_amd_evsel = a->pm_md.pm_amd.pm_amd_config;
-		PMCDBG2(MDP, ALL, 2,"amd-allocate ri=%d -> config=0x%x", ri,
-		    a->pm_md.pm_amd.pm_amd_config);
+		config = a->pm_md.pm_amd.pm_amd_config;
+		if ((config & ~amd_config_mask(amd_pmcdesc[ri].pm_subclass,
+		    caps)) != 0)
+			return (EINVAL);
+		pm->pm_md.pm_amd.pm_amd_evsel = config;
+		PMCDBG2(MDP, ALL, 2, "amd-allocate ri=%d -> config=0x%jx",
+		    ri, (uintmax_t)config);
 		return (0);
 	}
 
@@ -981,6 +1024,8 @@ pmc_amd_initialize(void)
 
 	pmc_mdep->pmd_npmc	+= amd_npmcs;
 
+	amd_init_policy();
+
 	PMCDBG0(MDP, INI, 0, "amd-initialize");
 
 	if (nclasses >= 3) {

sys/dev/hwpmc/hwpmc_amd.h

@@ -122,6 +122,18 @@
 #define	AMD_PMC_L3_TO_UNITMASK(x)	(((x) << 8) & AMD_PMC_UNITMASK)
 #define	AMD_PMC_L3_TO_EVENTMASK(x)	((x) & 0xFF)
 
+#define	AMD_PMC_L3_FAMILY17_MASK					\
+	(AMD_PMC_ENABLE | AMD_PMC_L3_TO_EVENTMASK(0xff) |		\
+	 AMD_PMC_L3_TO_UNITMASK(0xff) |					\
+	 AMD_PMC_L31_SLICEMASK | AMD_PMC_L31_COREMASK)
+
+#define	AMD_PMC_L3_FAMILY19_MASK					\
+	(AMD_PMC_ENABLE | AMD_PMC_L3_TO_EVENTMASK(0xff) |		\
+	 AMD_PMC_L3_TO_UNITMASK(0xff) |					\
+	 AMD_PMC_L32_THREADMASK | AMD_PMC_L32_SOURCEMASK |		\
+	 AMD_PMC_L32_ALLCORES | AMD_PMC_L32_ALLSOURCES |		\
+	 AMD_PMC_L32_COREMASK)
+
 #define AMD_PMC_L3_CAPS		(PMC_CAP_READ | PMC_CAP_WRITE | \
 	PMC_CAP_QUALIFIER | PMC_CAP_DOMWIDE)
 
@@ -148,6 +160,16 @@
 #define AMD_PMC_DF2_TO_UNITMASK(x)	((((x) & 0xFF) << 8) | \
 	(((uint64_t)(x) & 0x0F00) << 16))
 
+#define	AMD_PMC_DF_FAMILY17_MASK					\
+	(AMD_PMC_ENABLE |						\
+	 AMD_PMC_DF1_TO_EVENTMASK(0x3fff) |				\
+	 AMD_PMC_DF1_TO_UNITMASK(0xff))
+
+#define	AMD_PMC_DF_FAMILY1A_MASK					\
+	(AMD_PMC_ENABLE |						\
+	 AMD_PMC_DF2_TO_EVENTMASK(0x7fff) |				\
+	 AMD_PMC_DF2_TO_UNITMASK(0xfff))
+
 #define	AMD_NPMCS_K8		4
 #define AMD_NPMCS_MAX		(AMD_PMC_CORE_MAX + AMD_PMC_L3_MAX + \
 				 AMD_PMC_DF_MAX)

sys/dev/hwpmc/hwpmc_ibs.c

@@ -57,6 +57,8 @@ struct ibs_descr {
  * Globals
  */
 static uint64_t ibs_features;
+static uint64_t ibs_fetch_allowed_mask;
+static uint64_t ibs_op_allowed_mask;
 
 /*
  * Per-processor information
@@ -71,6 +73,73 @@ struct ibs_cpu {
 };
 static struct ibs_cpu **ibs_pcpu;
 
+static void
+ibs_init_policy(void)
+{
+
+	ibs_fetch_allowed_mask = IBS_FETCH_ALLOWED_MASK_BASE;
+
+	ibs_op_allowed_mask = IBS_OP_CTL_MAXCNTBASEMASK;
+
+	if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0)
+		ibs_fetch_allowed_mask |= IBS_FETCH_CTL_L3MISSONLY;
+
+	if ((ibs_features & CPUID_IBSID_OPCNT) != 0)
+		ibs_op_allowed_mask |= IBS_OP_CTL_COUNTERCONTROL;
+
+	if ((ibs_features & CPUID_IBSID_OPCNTEXT) != 0)
+		ibs_op_allowed_mask |= IBS_OP_CTL_MAXCNTEXTMASK;
+
+	if ((ibs_features & CPUID_IBSID_ZEN4IBSEXTENSIONS) != 0)
+		ibs_op_allowed_mask |= IBS_OP_CTL_L3MISSONLY;
+}
+
+static int
+ibs_validate_fetch_config(uint64_t config)
+{
+
+	if ((config & ~ibs_fetch_allowed_mask) != 0)
+		return (EINVAL);
+
+	return (0);
+}
+
+static int
+ibs_validate_op_config(uint64_t config)
+{
+	uint64_t allowed_mask;
+
+	allowed_mask = ibs_op_allowed_mask;
+
+	if ((config & IBS_OP_CTL_LATFLTEN) != 0) {
+		if ((ibs_features & CPUID_IBSID_IBSLOADLATENCYFILT) == 0)
+			return (EINVAL);
+		if ((config & IBS_OP_CTL_L3MISSONLY) == 0)
+			return (EINVAL);
+
+		allowed_mask |= IBS_OP_CTL_LDLATMASK | IBS_OP_CTL_L3MISSONLY;
+	}
+
+	if ((config & ~allowed_mask) != 0)
+		return (EINVAL);
+
+	return (0);
+}
+
+static int
+ibs_validate_pmc_config(int ri, uint64_t config)
+{
+
+	switch (ri) {
+	case IBS_PMC_FETCH:
+		return (ibs_validate_fetch_config(config));
+	case IBS_PMC_OP:
+		return (ibs_validate_op_config(config));
+	default:
+		return (EINVAL);
+	}
+}
+
 /*
  * Read a PMC value from the MSR.
  */
@@ -182,6 +251,7 @@ ibs_allocate_pmc(int cpu __unused, int ri, struct pmc *pm,
     const struct pmc_op_pmcallocate *a)
 {
 	uint64_t caps, config;
+	int error;
 
 	KASSERT(ri >= 0 && ri < IBS_NPMCS,
 	    ("[ibs,%d] illegal row index %d", __LINE__, ri));
@@ -205,9 +275,13 @@ ibs_allocate_pmc(int cpu __unused, int ri, struct pmc *pm,
 		return (EINVAL);
 
 	config = a->pm_md.pm_ibs.ibs_ctl;
+	error = ibs_validate_pmc_config(ri, config);
+	if (error != 0)
+		return (error);
 	pm->pm_md.pm_ibs.ibs_ctl = config;
 
-	PMCDBG2(MDP, ALL, 2, "ibs-allocate ri=%d -> config=0x%x", ri, config);
+	PMCDBG2(MDP, ALL, 2, "ibs-allocate ri=%d -> config=0x%jx", ri,
+	    config);
 
 	return (0);
 }
@@ -361,7 +435,6 @@ pmc_ibs_process_fetch(struct pmc *pm, struct trapframe *tf, uint64_t config)
 	if ((ibs_features & CPUID_IBSID_IBSFETCHCTLEXTD) != 0) {
 		mpd.pl_mpdata[PMC_MPIDX_FETCH_EXTCTL] = rdmsr(IBS_FETCH_EXTCTL);
 	}
-	mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL] = config;
 	mpd.pl_mpdata[PMC_MPIDX_FETCH_LINADDR] = rdmsr(IBS_FETCH_LINADDR);
 	if ((config & IBS_FETCH_CTL_PHYSADDRVALID) != 0) {
 		mpd.pl_mpdata[PMC_MPIDX_FETCH_PHYSADDR] =
@@ -622,10 +695,14 @@ pmc_ibs_initialize(struct pmc_mdep *pmc_mdep, int ncpus)
 	if (cpu_exthigh >= CPUID_IBSID) {
 		do_cpuid(CPUID_IBSID, regs);
 		ibs_features = regs[0];
+		if ((ibs_features & CPUID_IBSID_IBSFFV) == 0)
+			ibs_features = 0;
 	} else {
 		ibs_features = 0;
 	}
 
+	ibs_init_policy();
+
 	PMCDBG0(MDP, INI, 0, "ibs-initialize");
 
 	return (0);

sys/dev/hwpmc/hwpmc_ibs.h

@@ -100,6 +100,8 @@
 #define IBS_FETCH_CTL_TO_LAT(_c)	(((_c) >> 32) & 0x0000FFFF)
 #define IBS_FETCH_COUNT_TO_CTL(_c)	(((_c) << 12) & IBS_FETCH_CTL_CURCNTMASK)
 #define IBS_FETCH_CTL_TO_COUNT(_c)	(((_c) & IBS_FETCH_CTL_CURCNTMASK) >> 12)
+#define IBS_FETCH_ALLOWED_MASK_BASE	(IBS_FETCH_CTL_MAXCNTMASK | \
+    IBS_FETCH_CTL_RANDOMIZE)
 
 #define IBS_FETCH_LINADDR		0xC0011031 /* Fetch Linear Address */
 #define IBS_FETCH_PHYSADDR		0xC0011032 /* Fetch Physical Address */
@@ -118,12 +120,22 @@
 #define IBS_OP_CTL_VALID		(1ULL << 18) /* Valid */
 #define IBS_OP_CTL_ENABLE		(1ULL << 17) /* Enable */
 #define IBS_OP_CTL_L3MISSONLY		(1ULL << 16) /* L3 Miss Filtering */
-#define IBS_OP_CTL_MAXCNTMASK		0x07F0FFFFULL
+#define IBS_OP_CTL_MAXCNTMASK		0x07F0FFFFULL /* Max Count */
+#define IBS_OP_CTL_MAXCNTEXTMASK	0x07F00000ULL /* Max Count Extended */
+#define IBS_OP_CTL_MAXCNTBASEMASK	(IBS_OP_CTL_MAXCNTMASK & \
+    ~IBS_OP_CTL_MAXCNTEXTMASK) /* Max Count Base */
 #define IBS_OP_CTL_CURCNTMASK		0x07FFFFFF00000000ULL
+#define IBS_OP_CTL_LDLATTRSHMASK	(0xFULL << 59) /* Load Lat Threshold */
+#define IBS_OP_CTL_LDLATMASK		(IBS_OP_CTL_LATFLTEN | \
+    IBS_OP_CTL_LDLATTRSHMASK) /* Load Lat Combined */
 
-#define IBS_OP_CTL_LDLAT_TO_CTL(_c)	((((ldlat) >> 7) - 1) << 59)
-#define IBS_OP_INTERVAL_TO_CTL(_c)	((((_c) >> 4) & 0x0000FFFFULL) | ((_c) & 0x07F00000))
-#define IBS_OP_CTL_TO_INTERVAL(_c)	((((_c) & 0x0000FFFFULL) << 4) | ((_c) & 0x07F00000))
+#define IBS_OP_CTL_LDLAT_TO_CTL(_c)	(((((_c) >> 7) - 1) & 0xFULL) << 59)
+#define IBS_OP_INTERVAL_TO_CTL(_c)					\
+	((((_c) >> 4) & IBS_OP_CTL_MAXCNTBASEMASK) |			\
+	((_c) & IBS_OP_CTL_MAXCNTEXTMASK))
+#define IBS_OP_CTL_TO_INTERVAL(_c)					\
+	((((_c) & IBS_OP_CTL_MAXCNTBASEMASK) << 4) |			\
+	((_c) & IBS_OP_CTL_MAXCNTEXTMASK))
 #define IBS_OP_COUNT_TO_CTL(_c)		(((_c) << 32) & IBS_OP_CTL_CURCNTMASK)
 #define IBS_OP_CTL_TO_COUNT(_c)		(((_c) & IBS_OP_CTL_CURCNTMASK) >> 32)