LinuxKPI: 802.11: when synching HT and VHT cap, mask rx_mcs

When we sync the sta data, mask the rx_mcs with what the hardware is
able to do so that we do not leave, e.g., a 2nd stream enabled on a 1x1
chipset.
iwlwifi(4) has a further check for the smps_mode to limit to NSS=1 but
I believe that is historic and not actually in use anymore.

This fixes firmware crashes on TLC updates with nss=1 but the nss=2 array
index also being populated (with HT/VHT80/160 mcs information):
data being populated:
  iwlwifi0: 0x20101A0D | ADVANCED_SYSASSERT
  iwlwifi0: 0x00000006 | umac data1
  iwlwifi0: 0x00000001 | umac data2
  iwlwifi0: 0x000003FF | umac data3
  iwlwifi0: 0x____050F | last host cmd

Reported by:	Claudio Zumbo (claudiozumbo gmail.com), Erik Power
Tested by:	Claudio Zumbo, Erik Power (eppower umich.edu)
PR:		290622
MFC after:	3 days
Sponsored by:	The FreeBSD Foundation

sys/compat/linuxkpi/common/src/linux_80211.c

@@ -128,11 +128,11 @@ SYSCTL_INT(_compat_linuxkpi_80211, OID_AUTO, debug, CTLFLAG_RWTUN,
 
 #define	UNIMPLEMENTED		if (linuxkpi_debug_80211 & D80211_TODO)		\
     printf("XXX-TODO %s:%d: UNIMPLEMENTED\n", __func__, __LINE__)
-#define	TRACEOK()		if (linuxkpi_debug_80211 & D80211_TRACEOK)	\
-    printf("XXX-TODO %s:%d: TRACEPOINT\n", __func__, __LINE__)
+#define	TRACEOK(_fmt, ...)	if (linuxkpi_debug_80211 & D80211_TRACEOK)	\
+    printf("%s:%d: TRACEPOINT " _fmt "\n", __func__, __LINE__, ##__VA_ARGS__)
 #else
 #define	UNIMPLEMENTED		do { } while (0)
-#define	TRACEOK()		do { } while (0)
+#define	TRACEOK(...)		do { } while (0)
 #endif
 
 /* #define	PREP_TX_INFO_DURATION	(IEEE80211_TRANS_WAIT * 1000) */
@@ -474,12 +474,14 @@ lkpi_sync_chanctx_cw_from_rx_bw(struct ieee80211_hw *hw,
 
 #if defined(LKPI_80211_HT)
 static void
-lkpi_sta_sync_ht_from_ni(struct ieee80211_vif *vif, struct ieee80211_sta *sta,
-    struct ieee80211_node *ni)
+lkpi_sta_sync_ht_from_ni(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+    struct ieee80211_sta *sta, struct ieee80211_node *ni)
 {
 	struct ieee80211vap *vap;
 	uint8_t *ie;
 	struct ieee80211_ht_cap *htcap;
+	struct ieee80211_sta_ht_cap *ht_cap, *sta_ht_cap;
+	enum nl80211_band band;
 	int i, rx_nss;
 
 	if ((ni->ni_flags & IEEE80211_NODE_HT) == 0) {
@@ -513,13 +515,23 @@ lkpi_sta_sync_ht_from_ni(struct ieee80211_vif *vif, struct ieee80211_sta *sta,
 	 * MCS sets from the Rx MCS Bitmask; then there is MCS 32 and
 	 * MCS33.. is UEQM.
 	 */
+	band = vif->bss_conf.chanctx_conf->def.chan->band;
+	ht_cap = &hw->wiphy->bands[band]->ht_cap;
+	sta_ht_cap = &sta->deflink.ht_cap;
 	rx_nss = 0;
 	for (i = 0; i < 4; i++) {
-		if (htcap->mcs.rx_mask[i] != 0)
+		TRACEOK("HT rx_mask[%d] sta %#04x & hw %#04x", i,
+		    sta_ht_cap->mcs.rx_mask[i], ht_cap->mcs.rx_mask[i]);
+		sta_ht_cap->mcs.rx_mask[i] =
+			sta_ht_cap->mcs.rx_mask[i] & ht_cap->mcs.rx_mask[i];
+		/* XXX-BZ masking unequal modulation? */
+
+		if (sta_ht_cap->mcs.rx_mask[i] != 0)
 			rx_nss++;
 	}
 	if (rx_nss > 0) {
-		sta->deflink.rx_nss = rx_nss;
+		TRACEOK("HT rx_nss = max(%d, %d)", rx_nss, sta->deflink.rx_nss);
+		sta->deflink.rx_nss = MAX(rx_nss, sta->deflink.rx_nss);
 	} else {
 		sta->deflink.ht_cap.ht_supported = false;
 		return;
@@ -548,14 +560,15 @@ lkpi_sta_sync_ht_from_ni(struct ieee80211_vif *vif, struct ieee80211_sta *sta,
 
 #if defined(LKPI_80211_VHT)
 static void
-lkpi_sta_sync_vht_from_ni(struct ieee80211_vif *vif, struct ieee80211_sta *sta,
-    struct ieee80211_node *ni)
+lkpi_sta_sync_vht_from_ni(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+    struct ieee80211_sta *sta, struct ieee80211_node *ni)
 {
+	struct ieee80211_sta_vht_cap *vht_cap, *sta_vht_cap;;
 	enum ieee80211_sta_rx_bandwidth bw;
+	enum nl80211_band band;
 	uint32_t width;
 	int rx_nss;
-	uint16_t rx_mcs_map;
-	uint8_t mcs;
+	uint16_t rx_map, tx_map;
 
 	if ((ni->ni_flags & IEEE80211_NODE_VHT) == 0 ||
 	    !IEEE80211_IS_CHAN_VHT_5GHZ(ni->ni_chan)) {
@@ -609,18 +622,49 @@ lkpi_sta_sync_vht_from_ni(struct ieee80211_vif *vif, struct ieee80211_sta *sta,
 	sta->deflink.bandwidth = bw;
 skip_bw:
 
+	band = vif->bss_conf.chanctx_conf->def.chan->band;
+	vht_cap = &hw->wiphy->bands[band]->vht_cap;
+	sta_vht_cap = &sta->deflink.vht_cap;
+
 	rx_nss = 0;
-	rx_mcs_map = sta->deflink.vht_cap.vht_mcs.rx_mcs_map;
+	rx_map = tx_map = 0;
 	for (int i = 7; i >= 0; i--) {
-		mcs = rx_mcs_map >> (2 * i);
-		mcs &= 0x3;
-		if (mcs != IEEE80211_VHT_MCS_NOT_SUPPORTED) {
-			rx_nss = i + 1;
-			break;
+		uint8_t card, sta;
+
+		card = (vht_cap->vht_mcs.rx_mcs_map >> (2 * i)) & 0x3;
+		sta  = (sta_vht_cap->vht_mcs.rx_mcs_map >> (2 * i)) & 0x3;
+		if (sta != IEEE80211_VHT_MCS_NOT_SUPPORTED) {
+			if (card == IEEE80211_VHT_MCS_NOT_SUPPORTED)
+				sta = IEEE80211_VHT_MCS_NOT_SUPPORTED;
+			else {
+				sta = MIN(sta, card);
+				rx_nss = i + 1;
+			}
 		}
+		rx_map |= (sta << (2 * i));
+
+		card = (vht_cap->vht_mcs.tx_mcs_map >> (2 * i)) & 0x3;
+		sta  = (sta_vht_cap->vht_mcs.tx_mcs_map >> (2 * i)) & 0x3;
+		if (sta != IEEE80211_VHT_MCS_NOT_SUPPORTED) {
+			if (card == IEEE80211_VHT_MCS_NOT_SUPPORTED)
+				sta = IEEE80211_VHT_MCS_NOT_SUPPORTED;
+			else
+				sta = MIN(sta, card);
+		}
+		tx_map |= (sta << (2 * i));
+	}
+	TRACEOK("VHT rx_mcs_map %#010x->%#010x, tx_mcs_map %#010x->%#010x, rx_nss = %d",
+	    sta_vht_cap->vht_mcs.rx_mcs_map, rx_map,
+	    sta_vht_cap->vht_mcs.tx_mcs_map, tx_map, rx_nss);
+	sta_vht_cap->vht_mcs.rx_mcs_map = rx_map;
+	sta_vht_cap->vht_mcs.tx_mcs_map = tx_map;
+	if (rx_nss > 0) {
+		TRACEOK("VHT rx_nss = max(%d, %d)", rx_nss, sta->deflink.rx_nss);
+		sta->deflink.rx_nss = MAX(rx_nss, sta->deflink.rx_nss);
+	} else {
+		sta->deflink.vht_cap.vht_supported = false;
+		return;
 	}
-	if (rx_nss > 0)
-		sta->deflink.rx_nss = rx_nss;
 
 	switch (sta->deflink.vht_cap.cap & IEEE80211_VHT_CAP_MAX_MPDU_MASK) {
 	case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454:
@@ -642,18 +686,18 @@ lkpi_sta_sync_from_ni(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
     struct ieee80211_sta *sta, struct ieee80211_node *ni, bool updchnctx)
 {
 
-#if defined(LKPI_80211_HT)
-	lkpi_sta_sync_ht_from_ni(vif, sta, ni);
-#endif
-#if defined(LKPI_80211_VHT)
-	lkpi_sta_sync_vht_from_ni(vif, sta, ni);
-#endif
-
 	/*
 	 * Ensure rx_nss is at least 1 as otherwise drivers run into
 	 * unexpected problems.
 	 */
-	sta->deflink.rx_nss = MAX(1, sta->deflink.rx_nss);
+	sta->deflink.rx_nss = 1;
+
+#if defined(LKPI_80211_HT)
+	lkpi_sta_sync_ht_from_ni(hw, vif, sta, ni);
+#endif
+#if defined(LKPI_80211_VHT)
+	lkpi_sta_sync_vht_from_ni(hw, vif, sta, ni);
+#endif
 
 	/*
 	 * We are also called from node allocation which net80211
@@ -6237,7 +6281,6 @@ lkpi_ic_getradiocaps_ht(struct ieee80211com *ic, struct ieee80211_hw *hw,
 #endif
 
 	IMPROVE("PS, ampdu_*, ht_cap.mcs.tx_params, ...");
-	ic->ic_htcaps |= IEEE80211_HTCAP_SMPS_OFF;
 
 	/* Only add HT40 channels if supported. */
 	if ((ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40) != 0 &&