bnxt_en: VF ring reservation, HWRM registration, and PF-only operation guards

VFs require separate HWRM commands for ring reservation and async
completion ring setup, so a common PF/VF dispatcher is introduced and
the async CR path is extended to handle both.  The PF must populate the
VF request forwarding bitmap during driver registration so the firmware
correctly forwards VF-originated HWRM commands.  VF reservation strategy
and min-guaranteed capability flags are now parsed for correct resource
partitioning, and PF-only operations (DCB, NVM, package version sysctl)
are guarded against VF invocation.

The short command buffer allocation is also reordered before the function
reset to ensure extended HWRM messages are available when needed, a
prerequisite uncovered during VF bring-up.

MFC after:      1 month
Reviewed by:    ssaxena
Differential Revision: https://reviews.freebsd.org/D56232

sys/dev/bnxt/bnxt_en/bnxt.h

@@ -1398,5 +1398,6 @@ int bnxt_dcb_ieee_delapp(struct bnxt_softc *softc, struct bnxt_dcb_app *app);
 int bnxt_dcb_ieee_listapp(struct bnxt_softc *softc, struct bnxt_dcb_app *app,
     size_t nitems, int *num_inputs);
 void bnxt_set_flags_by_devid(struct bnxt_softc *softc);
+int bnxt_hwrm_reserve_rings(struct bnxt_softc *softc);
 
 #endif /* _BNXT_H */

sys/dev/bnxt/bnxt_en/bnxt_dcb.c

@@ -326,6 +326,9 @@ bnxt_hwrm_get_dcbx_app(struct bnxt_softc *softc, struct bnxt_dcb_app *app,
 	if (softc->hwrm_spec_code < 0x10601)
 		return 0;
 
+	if (BNXT_VF(softc))
+		return 0;
+
 	bnxt_hwrm_cmd_hdr_init(softc, &get, HWRM_FW_GET_STRUCTURED_DATA);
 
 	n = BNXT_IEEE_8021QAZ_MAX_TCS;

sys/dev/bnxt/bnxt_en/bnxt_hwrm.c

@@ -79,6 +79,14 @@ long bnxt_rx_pkts_pri_arr_base_off[] = {BNXT_RX_STATS_PRI_ENTRIES(rx_packets)};
 long bnxt_tx_bytes_pri_arr_base_off[] = {BNXT_TX_STATS_PRI_ENTRIES(tx_bytes)};
 long bnxt_tx_pkts_pri_arr_base_off[] = {BNXT_TX_STATS_PRI_ENTRIES(tx_packets)};
 
+uint16_t bnxt_vf_req_snif[] = {
+	HWRM_FUNC_CFG,
+	HWRM_FUNC_VF_CFG,
+	HWRM_PORT_PHY_QCFG,
+	HWRM_CFA_L2_FILTER_ALLOC,
+	HWRM_OEM_CMD,
+};
+
 static int
 bnxt_hwrm_err_map(uint16_t err)
 {
@@ -757,10 +765,22 @@ int bnxt_hwrm_func_resc_qcaps(struct bnxt_softc *softc, bool all)
 	hw_resc->max_stat_ctxs = le16toh(resp->max_stat_ctx);
 
 	if (BNXT_CHIP_P5_PLUS(softc)) {
-		hw_resc->max_nqs = le16toh(resp->max_msix);
+		hw_resc->max_nqs = hw_resc->max_irqs = le16toh(resp->max_msix);
 		hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
 	}
 	
+	if (BNXT_PF(softc)) {
+		struct bnxt_pf_info *pf = &softc->pf;
+
+		pf->vf_resv_strategy = le16toh(resp->vf_reservation_strategy);
+		if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
+			pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
+
+		if (resp->flags &
+		    htole16(FUNC_RESOURCE_QCAPS_RESP_FLAGS_MIN_GUARANTEED))
+			softc->fw_cap |= BNXT_FW_CAP_VF_RES_MIN_GUARANTEED;
+	}
+
 hwrm_func_resc_qcaps_exit:
 	BNXT_HWRM_UNLOCK(softc);
 	return rc;
@@ -1054,6 +1074,26 @@ int bnxt_hwrm_func_drv_rgtr(struct bnxt_softc *bp, unsigned long *bmap, int bmap
 	req.os_type = htole16(HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_FREEBSD);
 
 	if (BNXT_PF(bp)) {
+		u32 data[8];
+		int i;
+
+		memset(data, 0, sizeof(data));
+		for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
+			u16 cmd = bnxt_vf_req_snif[i];
+			unsigned int bit, idx;
+
+			if ((bp->fw_cap & BNXT_FW_CAP_LINK_ADMIN) &&
+			    (cmd == HWRM_PORT_PHY_QCFG))
+				continue;
+
+			idx = cmd / 32;
+			bit = cmd % 32;
+			data[idx] |= 1 << bit;
+		}
+
+		for (i = 0; i < 8; i++)
+			req.vf_req_fwd[i] = cpu_to_le32(data[i]);
+
 		req.enables |=
 			htole32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VF_REQ_FWD);
 	}
@@ -2299,14 +2339,38 @@ bnxt_hwrm_reserve_pf_rings(struct bnxt_softc *softc)
 	return hwrm_send_message(softc, &req, sizeof(req));
 }
 
+#define BNXT_VF_MAX_L2_CTX      4
+int bnxt_hwrm_reserve_vf_rings(struct bnxt_softc *softc)
+{
+	struct hwrm_func_vf_cfg_input req = {0};
+
+	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_VF_CFG);
+
+	req.enables |= htole32(HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_RSSCOS_CTXS);
+	req.enables |= htole32(HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_CMPL_RINGS);
+	req.enables |= htole32(HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_TX_RINGS);
+	req.enables |= htole32(HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_RX_RINGS);
+	req.enables |= htole32(HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_VNICS);
+	req.enables |= htole32(HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_STAT_CTXS);
+	req.enables |= htole32(HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_L2_CTXS);
+	req.num_rsscos_ctxs = htole16(0x8);
+	req.num_cmpl_rings = htole16(BNXT_MAX_NUM_QUEUES * 2);
+	req.num_tx_rings = htole16(BNXT_MAX_NUM_QUEUES);
+	req.num_rx_rings = htole16(BNXT_MAX_NUM_QUEUES);
+	req.num_vnics = htole16(BNXT_MAX_NUM_QUEUES);
+	req.num_stat_ctxs = htole16(BNXT_MAX_NUM_QUEUES * 2);
+	req.num_l2_ctxs = htole16(BNXT_VF_MAX_L2_CTX);
+
+	return hwrm_send_message(softc, &req, sizeof(req));
+}
+
 int
 bnxt_cfg_async_cr(struct bnxt_softc *softc)
 {
 	int rc = 0;
-	struct hwrm_func_cfg_input req = {0};
 
-	if (!BNXT_PF(softc))
+	if (BNXT_PF(softc)) {
-		return 0;
+		struct hwrm_func_cfg_input req = {0};
 
 		bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG);
 
@@ -2318,6 +2382,20 @@ bnxt_cfg_async_cr(struct bnxt_softc *softc)
 			req.async_event_cr = htole16(softc->def_cp_ring.ring.phys_id);
 
 		rc = hwrm_send_message(softc, &req, sizeof(req));
+	} else {
+		/* VF needs to configure async event completion ring using HWRM_FUNC_VF_CFG */
+		struct hwrm_func_vf_cfg_input req = {0};
+
+		bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_VF_CFG);
+
+		req.enables = htole32(HWRM_FUNC_VF_CFG_INPUT_ENABLES_ASYNC_EVENT_CR);
+		if (BNXT_CHIP_P5_PLUS(softc))
+			req.async_event_cr = htole16(softc->nq_rings[0].ring.phys_id);
+		else
+			req.async_event_cr = htole16(softc->def_cp_ring.ring.phys_id);
+
+		rc = hwrm_send_message(softc, &req, sizeof(req));
+	}
 
 	return rc;
 }
@@ -2392,6 +2470,9 @@ bnxt_hwrm_nvm_find_dir_entry(struct bnxt_softc *softc, uint16_t type,
 	int	rc = 0;
 	uint32_t old_timeo;
 	
+	if (BNXT_VF(softc))
+		return 0;
+
 	MPASS(ordinal);
 
 	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_FIND_DIR_ENTRY);

sys/dev/bnxt/bnxt_en/bnxt_hwrm.h

@@ -141,4 +141,5 @@ int bnxt_hwrm_passthrough(struct bnxt_softc *softc, void *req, uint32_t req_len,
 int	_hwrm_send_message(struct bnxt_softc *, void *, uint32_t);
 int	hwrm_send_message(struct bnxt_softc *, void *, uint32_t);
 void	bnxt_hwrm_cmd_hdr_init(struct bnxt_softc *, void *, uint16_t);
+int bnxt_hwrm_reserve_vf_rings (struct bnxt_softc *softc);
 #endif

sys/dev/bnxt/bnxt_en/bnxt_sysctl.c

@@ -1219,6 +1219,7 @@ bnxt_create_ver_sysctls(struct bnxt_softc *softc)
 	    "chip_type", CTLFLAG_RD, vi->chip_type > MAX_CHIP_TYPE ?
 	    bnxt_chip_type[MAX_CHIP_TYPE] : bnxt_chip_type[vi->chip_type], 0,
 	    "RoCE firmware name");
+	if (!BNXT_VF(softc))
 		SYSCTL_ADD_PROC(&vi->ver_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
 		    "package_ver", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
 		    softc, 0, bnxt_package_ver_sysctl, "A",

sys/dev/bnxt/bnxt_en/if_bnxt.c

@@ -2197,7 +2197,8 @@ static int bnxt_open(struct bnxt_softc *bp)
 	}
 
 	if (BNXT_CHIP_P5_PLUS(bp))
-		bnxt_hwrm_reserve_pf_rings(bp);
+		bnxt_hwrm_reserve_rings(bp);
+	
 	/* Get the current configuration of this function */
 	rc = bnxt_hwrm_func_qcfg(bp);
 	if (rc) {
@@ -2435,6 +2436,16 @@ static void bnxt_sp_task(struct work_struct *work)
 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
 }
 
+int
+bnxt_hwrm_reserve_rings(struct bnxt_softc *softc)
+{
+	if (BNXT_PF(softc))
+		return bnxt_hwrm_reserve_pf_rings(softc);
+
+	else
+		return bnxt_hwrm_reserve_vf_rings(softc);
+}
+
 /* Device setup and teardown */
 static int
 bnxt_attach_pre(if_ctx_t ctx)
@@ -2510,9 +2521,6 @@ bnxt_attach_pre(if_ctx_t ctx)
 		goto ver_fail;
 	}
 
-	/* Now perform a function reset */
-	rc = bnxt_hwrm_func_reset(softc);
-
 	if ((softc->flags & BNXT_FLAG_SHORT_CMD) ||
 	    softc->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
 		rc = bnxt_alloc_hwrm_short_cmd_req(softc);
@@ -2520,6 +2528,9 @@ bnxt_attach_pre(if_ctx_t ctx)
 			goto hwrm_short_cmd_alloc_fail;
 	}
 	
+	/* Now perform a function reset */
+	rc = bnxt_hwrm_func_reset(softc);
+
 	if ((softc->ver_info->chip_num == BCM57508) ||
 	    (softc->ver_info->chip_num == BCM57504) ||
 	    (softc->ver_info->chip_num == BCM57504_NPAR) ||
@@ -2670,8 +2681,10 @@ bnxt_attach_pre(if_ctx_t ctx)
 
 	/* Get the queue config */
 	bnxt_get_wol_settings(softc);
+
 	if (BNXT_CHIP_P5_PLUS(softc))
-		bnxt_hwrm_reserve_pf_rings(softc);
+		bnxt_hwrm_reserve_rings(softc);
+
 	rc = bnxt_hwrm_func_qcfg(softc);
 	if (rc) {
 		device_printf(softc->dev, "attach: hwrm func qcfg failed\n");