bnxt_en: Add core SR-IOV infrastructure

Introduce the foundational building blocks for SR-IOV Virtual Function
support on Broadcom NetXtreme-C/E adapters.

* Add bnxt_sriov.h: defines the extended bnxt_vf_info structure (per-VF
  firmware FID, MAC addresses, VLAN, flags, DMA command buffers, resource
  counts), the bnxt_resc_map helper, flag macros (BNXT_VF_TRUST,
  BNXT_VF_SPOOFCHK, etc.), and prototypes for all SR-IOV functions.

* Add bnxt_sriov.c: implements the SR-IOV attachment sequence
  (bnxt_sriov_attach), the iflib IOV callbacks (bnxt_iov_init,
  bnxt_iov_uninit, bnxt_iov_vf_add), VF resource allocation and
  firmware configuration helpers (bnxt_alloc_vf_resources,
  bnxt_cfg_hw_sriov, bnxt_hwrm_func_vf_resc_cfg, bnxt_hwrm_func_buf_rgtr,
  bnxt_hwrm_func_vf_resource_free), and the per-VF parameter helper.

* Extend bnxt.h: include bnxt_sriov.h; extend bnxt_pf_info with VF-
  tracking fields (vf array, firmware FID/MAC, resource-reservation
  strategy, DMA page management, sysctl context); replace the upstream
  bnxt_vf_info stub with the full definition from bnxt_sriov.h; extend
  bnxt_func_qcfg with allocation counters required by the VF resource
  configuration path; add vf_resc_cfg_input and sriov_lock to bnxt_softc.

* Update Makefile to build bnxt_sriov.c and include bnxt_sriov.h.

* Wire up PCI-IOV device methods (pci_iov_init / pci_iov_uninit /
  pci_iov_add_vf) and iflib IOV callbacks (ifdi_iov_init / ifdi_iov_uninit
  / ifdi_iov_vf_add) in if_bnxt.c; call bnxt_sriov_attach() from
  bnxt_attach_post() on P5+ Physical Functions.

MFC after:      1 month
Reviewed by:    ssaxena
Differential Revision: https://reviews.freebsd.org/D56197

sys/dev/bnxt/bnxt_en/bnxt.h

@@ -47,6 +47,7 @@
 #include "hsi_struct_def.h"
 #include "bnxt_dcb.h"
 #include "bnxt_auxbus_compat.h"
+#include "bnxt_sriov.h"
 
 #define DFLT_HWRM_CMD_TIMEOUT		500
 
@@ -95,9 +96,18 @@
 #define NETXTREME_C_VF1	0x16cb
 #define NETXTREME_C_VF2	0x16e1
 #define NETXTREME_C_VF3	0x16e5
-#define NETXTREME_E_VF1	0x16c1
-#define NETXTREME_E_VF2	0x16d3
-#define NETXTREME_E_VF3	0x16dc
+
+#define NETXTREME_E_VF1	0x1606
+#define NETXTREME_E_VF2	0x1609
+#define NETXTREME_E_VF3	0x16c1
+#define NETXTREME_E_VF4	0x16d3
+#define NETXTREME_E_VF5	0x16dc
+
+#define NETXTREME_E_P5_VF1	0x1806
+#define NETXTREME_E_P5_VF2	0x1807
+#define NETXTREME_E_P5_VF_HV1	0x1808
+#define NETXTREME_E_P5_VF_HV2	0x1809
+#define NETXTREME_E_P7_VF	0x1819
 
 #define EVENT_DATA1_RESET_NOTIFY_FATAL(data1)				\
 	(((data1) &							\
@@ -504,7 +514,9 @@ struct bnxt_pf_info {
 	uint8_t		port_id;
 	uint32_t	first_vf_id;
 	uint16_t	active_vfs;
+	uint16_t	registered_vfs;
 	uint16_t	max_vfs;
+	uint16_t	max_msix_vfs;
 	uint32_t	max_encap_records;
 	uint32_t	max_decap_records;
 	uint32_t	max_tx_em_flows;
@@ -515,31 +527,14 @@ struct bnxt_pf_info {
 	uint16_t	hwrm_cmd_req_pages;
 	void		*hwrm_cmd_req_addr[4];
 	bus_addr_t	hwrm_cmd_req_dma_addr[4];
-};
-
-struct bnxt_vf_info {
 	uint16_t	fw_fid;
 	uint8_t		mac_addr[ETHER_ADDR_LEN];
-	uint16_t	max_rsscos_ctxs;
-	uint16_t	max_cp_rings;
-	uint16_t	max_tx_rings;
-	uint16_t	max_rx_rings;
-	uint16_t	max_hw_ring_grps;
-	uint16_t	max_l2_ctxs;
-	uint16_t	max_irqs;
-	uint16_t	max_vnics;
-	uint16_t	max_stat_ctxs;
-	uint32_t	vlan;
-#define BNXT_VF_QOS		0x1
-#define BNXT_VF_SPOOFCHK	0x2
-#define BNXT_VF_LINK_FORCED	0x4
-#define BNXT_VF_LINK_UP		0x8
-	uint32_t	flags;
-	uint32_t	func_flags; /* func cfg flags */
-	uint32_t	min_tx_rate;
-	uint32_t	max_tx_rate;
-	void		*hwrm_cmd_req_addr;
-	bus_addr_t	hwrm_cmd_req_dma_addr;
+	uint8_t		vf_resv_strategy;
+	uint8_t		num_vfs;
+	struct		bnxt_vf_info *vf;
+	uint16_t	vf_hwrm_cmd_req_page_shift;
+	struct sysctl_ctx_list	sysctl_ctx;
+	struct sysctl_oid	*sysctl_node;
 };
 
 #define BNXT_PF(softc)		(!((softc)->flags & BNXT_FLAG_VF))
@@ -698,6 +693,12 @@ struct bnxt_func_qcfg {
 	uint16_t alloc_tx_rings;
 	uint16_t alloc_rx_rings;
 	uint16_t alloc_vnics;
+	uint16_t alloc_rss_ctx;
+	uint16_t alloc_l2_ctx;
+	uint16_t alloc_vfs;
+	uint16_t alloc_hw_ring_grps;
+	uint16_t alloc_stat_ctx;
+	uint16_t alloc_msix;
 };
 
 struct bnxt_hw_lro {
@@ -1089,6 +1090,7 @@ struct bnxt_softc {
 	struct bnxt_func_qcfg	fn_qcfg;
 	struct bnxt_pf_info	pf;
 	struct bnxt_vf_info	vf;
+	struct hwrm_func_vf_resource_cfg_input vf_resc_cfg_input;
 
 	uint16_t		hwrm_cmd_seq;
 	uint32_t		hwrm_cmd_timeo;	/* milliseconds */
@@ -1097,6 +1099,7 @@ struct bnxt_softc {
 	/* Interrupt info for HWRM */
 	struct if_irq		irq;
 	struct mtx		hwrm_lock;
+	struct mtx		sriov_lock;
 	uint16_t		hwrm_max_req_len;
 	uint16_t		hwrm_max_ext_req_len;
 	uint32_t		hwrm_spec_code;

sys/dev/bnxt/bnxt_en/bnxt_sriov.c

@@ -0,0 +1,469 @@
+#include <linux/delay.h>
+#include <linux/etherdevice.h>
+
+#include "opt_global.h"
+#include "bnxt.h"
+#include "hsi_struct_def.h"
+#include "bnxt_hwrm.h"
+#include "bnxt_sriov.h"
+
+
+static int
+bnxt_set_vf_admin_mac(struct bnxt_softc *softc, struct bnxt_vf_info *vf,
+		      const uint8_t *mac)
+{
+	struct hwrm_func_cfg_input req = {0};
+	int rc;
+
+	if (!BNXT_PF(softc))
+		return (EOPNOTSUPP);
+
+	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG);
+
+	req.fid = htole16(vf->fw_fid);
+	req.enables = htole32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR);
+	memcpy(req.dflt_mac_addr, mac, ETHER_ADDR_LEN);
+
+	BNXT_HWRM_LOCK(softc);
+	rc = _hwrm_send_message(softc, &req, sizeof(req));
+	BNXT_HWRM_UNLOCK(softc);
+
+	return (rc);
+}
+
+static void
+bnxt_vf_parse_schema(struct bnxt_softc *softc, struct bnxt_vf_info *vf,
+		     const nvlist_t *params)
+{
+	const void *mac;
+	size_t maclen;
+
+	memset(vf->mac_addr, 0, ETHER_ADDR_LEN);
+	memset(vf->vf_mac_addr, 0, ETHER_ADDR_LEN);
+
+	if (params == NULL)
+		return;
+
+	if (nvlist_exists(params, "mac-anti-spoof"))
+		vf->spoofchk = nvlist_get_bool(params, "mac-anti-spoof");
+	if (nvlist_exists(params, "trust"))
+		vf->trusted = nvlist_get_bool(params, "trust");
+
+	if (!nvlist_exists(params, "mac-addr"))
+		return;
+
+	mac = nvlist_get_binary(params, "mac-addr", &maclen);
+
+	if (maclen != ETHER_ADDR_LEN)
+		return;
+
+	if (!is_valid_ether_addr(mac))
+		return;
+
+	memcpy(vf->mac_addr, mac, ETHER_ADDR_LEN);
+	vf->has_admin_mac = true;
+}
+
+/* Add a Virtual Functions */
+int
+bnxt_iov_vf_add(if_ctx_t ctx, uint16_t vfnum, const nvlist_t *params)
+{
+	struct bnxt_softc *softc = iflib_get_softc(ctx);
+	struct bnxt_vf_info *vf = &softc->pf.vf[vfnum];
+	int rc;
+
+	vf->fw_fid = softc->pf.first_vf_id + vfnum;
+	vf->vfnum = vfnum;
+
+	/* Parse schema */
+	bnxt_vf_parse_schema(softc, vf, params);
+
+	/*
+	 * If user provided MAC, program it into firmware.
+	 */
+	if (vf->has_admin_mac) {
+		rc = bnxt_set_vf_admin_mac(softc, vf, vf->mac_addr);
+		if (rc)
+			device_printf(softc->dev,
+				      "vf%u: PF-assigned MAC programming failed (rc=%d), falling back to firmware/default MAC\n",
+				      vfnum, rc);
+	}
+
+	return 0;
+}
+
+/* Free driver-side VF resources (called after hwrm_vf_resc_free) */
+void bnxt_free_vf_resources(struct bnxt_softc *softc)
+{
+	int i;
+	size_t page_size = 1UL << softc->pf.vf_hwrm_cmd_req_page_shift;
+
+	softc->pf.active_vfs = 0;
+
+	if (softc->pf.vf) {
+		kfree(softc->pf.vf);
+		softc->pf.vf = NULL;
+	}
+	if (softc->pf.vf_event_bmap) {
+		kfree(softc->pf.vf_event_bmap);
+		softc->pf.vf_event_bmap = NULL;
+	}
+	for (i = 0; i < softc->pf.hwrm_cmd_req_pages; i++) {
+		if (softc->pf.hwrm_cmd_req_addr[i]) {
+			dma_free_coherent(&softc->pdev->dev, page_size,
+					  softc->pf.hwrm_cmd_req_addr[i],
+					  softc->pf.hwrm_cmd_req_dma_addr[i]);
+					  softc->pf.hwrm_cmd_req_addr[i] = NULL;
+		}
+	}
+}
+
+/* Free firmware-side VF resources */
+int
+bnxt_hwrm_func_vf_resource_free(struct bnxt_softc *softc, int num_vfs)
+{
+	int i, rc;
+	struct hwrm_func_vf_resc_free_input req;
+
+	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_VF_RESC_FREE);
+
+	BNXT_HWRM_LOCK(softc);
+	for (i = softc->pf.first_vf_id; i < softc->pf.first_vf_id + num_vfs; i++) {
+		req.vf_id = cpu_to_le16(i);
+		rc = _hwrm_send_message(softc, &req, sizeof(req));
+		if (rc)
+			break;
+	}
+	BNXT_HWRM_UNLOCK(softc);
+
+	return rc;
+}
+
+/* Free all VF resources */
+void bnxt_iov_uninit(if_ctx_t ctx)
+{
+	int rc;
+	struct bnxt_softc *softc = iflib_get_softc(ctx);
+	int num_vfs = softc->pf.num_vfs;
+
+	if (!num_vfs)
+		return;
+
+	BNXT_SRIOV_LOCK(softc);
+	softc->pf.num_vfs = 0;
+	BNXT_SRIOV_UNLOCK(softc);
+
+	rc = bnxt_hwrm_func_vf_resource_free(softc, num_vfs);
+	if (rc)
+		device_printf(softc->dev, "VF resource free HWRM failed: %d\n", rc);
+
+	bnxt_free_vf_resources(softc);
+	BNXT_SRIOV_LOCK_DESTROY(softc);
+}
+
+static inline int
+bnxt_set_vf_resc_field(uint16_t *min_field, uint16_t *max_field,
+		       uint16_t hw_max, uint16_t pf_alloc, int num_vfs)
+{
+	uint16_t val = 0;
+
+	if (num_vfs <= 0)
+		return -EINVAL;
+
+	if (hw_max > pf_alloc)
+		val = (hw_max - pf_alloc) / num_vfs;
+
+	*min_field = *max_field = cpu_to_le16(val);
+
+	return 0;
+}
+
+static int bnxt_set_vf_params(struct bnxt_softc *softc, int vf_id)
+{
+	struct hwrm_func_cfg_input req = {0};
+	struct bnxt_vf_info *vf;
+	int rc;
+
+	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG);
+
+	vf = &softc->pf.vf[vf_id];
+	req.fid = cpu_to_le16(vf->fw_fid);
+
+
+	if (is_valid_ether_addr(vf->mac_addr)) {
+		req.enables |= cpu_to_le32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR);
+		memcpy(req.dflt_mac_addr, vf->mac_addr, ETHER_ADDR_LEN);
+	}
+
+	if (vf->vlan) {
+		req.enables |= cpu_to_le32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN);
+		req.dflt_vlan = cpu_to_le16(vf->vlan);
+	}
+
+	if (vf->flags & BNXT_VF_TRUST)
+		req.flags = cpu_to_le32(HWRM_FUNC_CFG_INPUT_FLAGS_TRUSTED_VF_ENABLE);
+
+	BNXT_HWRM_LOCK(softc);
+	rc = _hwrm_send_message(softc, &req, sizeof(req));
+	BNXT_HWRM_UNLOCK(softc);
+	if (rc)
+		device_printf(softc->dev, "hwrm_func_cfg failed (error:%d)\n", rc);
+
+	return rc;
+}
+
+static int
+bnxt_hwrm_func_vf_resc_cfg(struct bnxt_softc *softc, int num_vfs, bool reset)
+{
+	struct hwrm_func_vf_resource_cfg_input req = {0};
+	struct bnxt_pf_info *pf = &softc->pf;
+	struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg;
+	struct bnxt_hw_resc *hw_resc = &softc->hw_resc;
+	int i, rc;
+	uint16_t msix_val = 0;
+
+	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_VF_RESOURCE_CFG);
+	struct bnxt_resc_map resc_table[] = {
+		{ &req.min_tx_rings,		&req.max_tx_rings,	hw_resc->max_tx_rings,		fn_qcfg->alloc_tx_rings },
+		{ &req.min_rx_rings,		&req.max_rx_rings,	hw_resc->max_rx_rings,		fn_qcfg->alloc_rx_rings },
+		{ &req.min_cmpl_rings,		&req.max_cmpl_rings,	hw_resc->max_cp_rings,		fn_qcfg->alloc_completion_rings },
+		{ &req.min_stat_ctx,		&req.max_stat_ctx,	hw_resc->max_stat_ctxs,		fn_qcfg->alloc_stat_ctx },
+		{ &req.min_vnics,		&req.max_vnics,		hw_resc->max_vnics,		fn_qcfg->alloc_vnics },
+		{ &req.min_hw_ring_grps,	&req.max_hw_ring_grps,	hw_resc->max_hw_ring_grps,	fn_qcfg->alloc_hw_ring_grps },
+		{ &req.min_rsscos_ctx,		&req.max_rsscos_ctx,	hw_resc->max_rsscos_ctxs,	fn_qcfg->alloc_rss_ctx },
+		{ &req.min_l2_ctxs,		&req.max_l2_ctxs,	hw_resc->max_l2_ctxs,		fn_qcfg->alloc_l2_ctx },
+	};
+
+	for (i = 0; i < sizeof(resc_table) / sizeof(resc_table[0]); i++) {
+		rc = bnxt_set_vf_resc_field(resc_table[i].min_field,
+					    resc_table[i].max_field,
+					    resc_table[i].hw_max,
+					    resc_table[i].pf_alloc,
+					    num_vfs);
+		if (rc)
+			return rc;
+	}
+
+	if (hw_resc->max_irqs > fn_qcfg->alloc_msix && num_vfs > 0)
+		msix_val = (hw_resc->max_irqs - fn_qcfg->alloc_msix) / num_vfs;
+
+	req.max_msix = cpu_to_le16(msix_val);
+
+	for (i = 0; i < num_vfs; i++) {
+		struct bnxt_vf_info *vf = &pf->vf[i];
+
+		vf->fw_fid = pf->first_vf_id + i;
+		if (reset) {
+			rc = bnxt_set_vf_params(softc, i);
+			if (rc)
+				break;
+		}
+
+		req.vf_id = cpu_to_le16(vf->fw_fid);
+
+		BNXT_HWRM_LOCK(softc);
+		rc = _hwrm_send_message(softc, &req, sizeof(req));
+		BNXT_HWRM_UNLOCK(softc);
+
+		if (rc) {
+			device_printf(softc->dev, "HWRM_FUNC_VF_RESOURCE_CFG req dump:\n");
+			break;
+		}
+
+		pf->active_vfs = i + 1;
+
+		vf->min_tx_rings    = le16_to_cpu(req.min_tx_rings);
+		vf->min_rx_rings    = le16_to_cpu(req.min_rx_rings);
+		vf->min_cp_rings    = le16_to_cpu(req.min_cmpl_rings);
+		vf->min_stat_ctxs   = le16_to_cpu(req.min_stat_ctx);
+		vf->min_ring_grps   = le16_to_cpu(req.min_hw_ring_grps);
+		vf->min_vnics       = le16_to_cpu(req.min_vnics);
+	}
+
+	if (pf->active_vfs)
+		memcpy(&softc->vf_resc_cfg_input, &req,
+		       sizeof(struct hwrm_func_vf_resource_cfg_input));
+
+	return rc;
+}
+
+static int
+bnxt_hwrm_func_buf_rgtr(struct bnxt_softc *softc)
+{
+	int rc;
+	struct hwrm_func_buf_rgtr_input req;
+
+	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_BUF_RGTR);
+
+	req.req_buf_num_pages = cpu_to_le16(softc->pf.hwrm_cmd_req_pages);
+	req.req_buf_page_size = cpu_to_le16(softc->pf.vf_hwrm_cmd_req_page_shift);
+	req.req_buf_len = cpu_to_le16(BNXT_HWRM_REQ_MAX_SIZE);
+	req.req_buf_page_addr0 = cpu_to_le64(softc->pf.hwrm_cmd_req_dma_addr[0]);
+	req.req_buf_page_addr1 = cpu_to_le64(softc->pf.hwrm_cmd_req_dma_addr[1]);
+	req.req_buf_page_addr2 = cpu_to_le64(softc->pf.hwrm_cmd_req_dma_addr[2]);
+	req.req_buf_page_addr3 = cpu_to_le64(softc->pf.hwrm_cmd_req_dma_addr[3]);
+
+	BNXT_HWRM_LOCK(softc);
+	rc = _hwrm_send_message(softc, &req, sizeof(req));
+	BNXT_HWRM_UNLOCK(softc);
+
+	return rc;
+}
+
+static void
+bnxt_set_vf_attr(struct bnxt_softc *softc, int num_vfs)
+{
+	int i;
+	struct bnxt_vf_info *vf;
+
+	for (i = 0; i < num_vfs; i++) {
+		vf = &softc->pf.vf[i];
+		memset(vf, 0, sizeof(*vf));
+	}
+}
+
+static int
+bnxt_alloc_vf_resources(struct bnxt_softc *softc, int num_vfs)
+{
+	struct pci_dev *pdev = softc->pdev;
+	u32 nr_pages, size, i, j, k = 0;
+	u32 page_size, reqs_per_page;
+	void *p;
+
+	p = kcalloc(num_vfs, sizeof(struct bnxt_vf_info), GFP_KERNEL);
+	if (!p)
+		return ENOMEM;
+
+	rcu_assign_pointer(softc->pf.vf, p);
+	bnxt_set_vf_attr(softc, num_vfs);
+
+	size = num_vfs * BNXT_HWRM_REQ_MAX_SIZE;
+	page_size = BNXT_PAGE_SIZE;
+	softc->pf.vf_hwrm_cmd_req_page_shift = BNXT_PAGE_SHIFT;
+	while (size > page_size * BNXT_MAX_VF_CMD_FWD_PAGES) {
+		page_size *= 2;
+		softc->pf.vf_hwrm_cmd_req_page_shift++;
+	}
+	nr_pages = DIV_ROUND_UP(size, page_size);
+	reqs_per_page = page_size / BNXT_HWRM_REQ_MAX_SIZE;
+
+	for (i = 0; i < nr_pages; i++) {
+		softc->pf.hwrm_cmd_req_addr[i] =
+			dma_alloc_coherent(&pdev->dev, page_size,
+					   &softc->pf.hwrm_cmd_req_dma_addr[i],
+					   GFP_ATOMIC);
+
+		if (!softc->pf.hwrm_cmd_req_addr[i])
+			return ENOMEM;
+
+		for (j = 0; j < reqs_per_page && k < num_vfs; j++) {
+			struct bnxt_vf_info *vf = &softc->pf.vf[k];
+
+			vf->hwrm_cmd_req_addr = (char *)softc->pf.hwrm_cmd_req_addr[i] +
+						j * BNXT_HWRM_REQ_MAX_SIZE;
+			vf->hwrm_cmd_req_dma_addr =
+				softc->pf.hwrm_cmd_req_dma_addr[i] + j *
+				BNXT_HWRM_REQ_MAX_SIZE;
+			k++;
+		}
+	}
+
+	softc->pf.vf_event_bmap = kzalloc(ALIGN(DIV_ROUND_UP(num_vfs, 8),
+					  sizeof(long)), GFP_ATOMIC);
+	if (!softc->pf.vf_event_bmap)
+		return ENOMEM;
+
+	softc->pf.hwrm_cmd_req_pages = nr_pages;
+
+	return 0;
+}
+
+int bnxt_cfg_hw_sriov(struct bnxt_softc *softc, uint16_t *num_vfs, bool reset)
+{
+	int rc;
+
+	rc = bnxt_hwrm_func_buf_rgtr(softc);
+	if (rc) {
+		device_printf(softc->dev, "hwrm func buf rgtr failed (error=%d)\n", rc);
+		return (EIO);
+	}
+
+	rc = bnxt_hwrm_func_vf_resc_cfg(softc, *num_vfs, reset);
+	if (rc) {
+		device_printf(softc->dev, "hwrm func VF resc config failed (error=%d)\n", rc);
+		return (EIO);
+	}
+
+	return (0);
+}
+
+int
+bnxt_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params)
+{
+	int rc;
+	if_t ifp = iflib_get_ifp(ctx);
+	struct bnxt_softc *softc = iflib_get_softc(ctx);
+	bool admin_up = !!(if_getflags(ifp) & IFF_UP);
+	bool running  = !!(if_getdrvflags(ifp) & IFF_DRV_RUNNING);
+
+	if (!admin_up || !running) {
+		device_printf(softc->dev, "PF is down, rejecting VF creation\n");
+		return ENETDOWN;
+	}
+
+	if (num_vfs > BNXT_MAX_VFS) {
+		device_printf(softc->dev, "Requested %u VFs exceeds maximum supported (%u)\n",
+			      num_vfs, BNXT_MAX_VFS);
+		return ERANGE;
+	}
+
+	/*
+	 * Initialize SR-IOV lock before creating any SR-IOV state, so sysctl/VF
+	 * paths can safely synchronize and error paths can always destroy it.
+	 */
+	BNXT_SRIOV_LOCK_INIT(softc, device_get_nameunit(softc->dev));
+
+	rc = bnxt_alloc_vf_resources(softc, num_vfs);
+	if (rc) {
+		device_printf(softc->dev, "VF resource alloc failed (error=%d)\n", rc);
+		goto fail_lock;
+	}
+
+	rc = bnxt_cfg_hw_sriov(softc, &num_vfs, false);
+	if (rc)
+		goto fail_free_vf_resc;
+
+
+	BNXT_SRIOV_LOCK(softc);
+	softc->pf.num_vfs = num_vfs;
+	BNXT_SRIOV_UNLOCK(softc);
+
+	return 0;
+
+fail_free_vf_resc:
+	bnxt_free_vf_resources(softc);
+fail_lock:
+	BNXT_SRIOV_LOCK_DESTROY(softc);
+	return rc;
+}
+
+void bnxt_sriov_attach(struct bnxt_softc *softc)
+{
+	int rc;
+	device_t dev = softc->dev;
+	nvlist_t *pf_schema, *vf_schema;
+
+	pf_schema = pci_iov_schema_alloc_node();
+	vf_schema = pci_iov_schema_alloc_node();
+
+	/* Optionally add VF-specific attributes to the VF schema */
+	pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
+	pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", IOV_SCHEMA_HASDEFAULT, FALSE);
+	pci_iov_schema_add_bool(vf_schema, "trust", IOV_SCHEMA_HASDEFAULT, FALSE);
+
+	/* Attach SR-IOV schemas to the device */
+	rc = pci_iov_attach(dev, pf_schema, vf_schema);
+	if (rc)
+		device_printf(dev, "Failed to initialize SR-IOV (error=%d)\n", rc);
+}
+

sys/dev/bnxt/bnxt_en/bnxt_sriov.h

@@ -0,0 +1,116 @@
+#ifndef _BNXT_SRIOV_H_
+#define _BNXT_SRIOV_H_
+
+#include <sys/iov_schema.h>
+#include <linux/pci.h>
+#include <dev/pci/pci_iov.h>
+
+#include "opt_global.h"
+#include "bnxt.h"
+
+#ifndef PCI_IOV
+#define PCI_IOV 1
+#endif
+
+/* macro definations */
+
+#define BNXT_MAX_VFS 4
+#define BNXT_HWRM_REQ_MAX_SIZE	128
+#define BNXT_MAX_VF_CMD_FWD_PAGES  4
+#define BNXT_VF_QOS		0x1
+#define BNXT_VF_SPOOFCHK	0x2
+#define BNXT_VF_LINK_FORCED	0x4
+#define BNXT_VF_LINK_UP		0x8
+#define BNXT_VF_TRUST		0x10
+#define BNXT_VLAN_VID_MASK	0x0fff
+
+#define BNXT_EXEC_FWD_RESP_SIZE_ERR(n)					\
+	((offsetof(struct hwrm_exec_fwd_resp_input, encap_request) + n) >\
+	 offsetof(struct hwrm_exec_fwd_resp_input, encap_resp_target_id))
+
+#define BNXT_VF_RESV_STRATEGY_MAXIMAL   0
+#define BNXT_VF_RESV_STRATEGY_MINIMAL   1
+#define BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC    2
+#define FUNC_RESOURCE_QCAPS_RESP_FLAGS_MIN_GUARANTEED     0x1UL
+
+#define BNXT_SRIOV_LOCK_INIT(sc, _name)	\
+	mtx_init(&(sc)->sriov_lock, _name, "sriov_lock", MTX_DEF | MTX_NOWITNESS)
+#define BNXT_SRIOV_LOCK(sc)		mtx_lock(&(sc)->sriov_lock)
+#define BNXT_SRIOV_UNLOCK(sc)		mtx_unlock(&(sc)->sriov_lock)
+#define BNXT_SRIOV_LOCK_DESTROY(sc)				\
+	do {							\
+		if (mtx_initialized(&(sc)->sriov_lock))		\
+			mtx_destroy(&(sc)->sriov_lock);		\
+} while (0)
+
+
+/* structure declartions/definations */
+
+struct bnxt_softc;
+
+struct bnxt_vf_info {
+	uint8_t		vfnum;
+	uint16_t	fw_fid;
+	uint8_t		mac_addr[ETHER_ADDR_LEN];
+	uint8_t		vf_mac_addr[ETHER_ADDR_LEN];
+	uint32_t	vlan;
+	uint32_t	flags;
+	uint32_t	func_qcfg_flags;
+	uint32_t	min_tx_rate;
+	uint32_t	max_tx_rate;
+	uint16_t	min_tx_rings;
+	uint16_t	max_tx_rings;
+	uint16_t	min_rx_rings;
+	uint16_t	max_rx_rings;
+	uint16_t	min_cp_rings;
+	uint16_t	max_cp_rings;
+	uint16_t	min_rsscos_ctxs;
+	uint16_t	max_rsscos_ctxs;
+	uint16_t	min_stat_ctxs;
+	uint16_t	max_stat_ctxs;
+	uint16_t	min_ring_grps;
+	uint16_t	max_hw_ring_grps;
+	uint16_t	min_vnics;
+	uint16_t	max_vnics;
+	uint16_t	min_irqs;
+	uint16_t	max_irqs;
+	uint16_t	min_l2_ctxs;
+	uint16_t	max_l2_ctxs;
+	void		*hwrm_cmd_req_addr;
+	bus_addr_t	hwrm_cmd_req_dma_addr;
+	struct		iflib_dma_info	hwrm_cmd_req;
+	uint16_t	trusted;
+	bool		spoofchk;
+	bool		has_admin_mac;
+};
+
+struct bnxt_resc_map {
+	uint16_t *min_field;
+	uint16_t *max_field;
+	uint16_t hw_max;
+	uint16_t pf_alloc;
+};
+
+/* function prototypes */
+
+void bnxt_sriov_attach(struct bnxt_softc *softc);
+int bnxt_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params);
+void bnxt_iov_uninit(if_ctx_t ctx);
+int bnxt_iov_vf_add(if_ctx_t ctx, uint16_t vfnum, const nvlist_t *params);
+int bnxt_hwrm_func_vf_resource_free(struct bnxt_softc *softc, int num_vfs);
+void bnxt_free_vf_resources(struct bnxt_softc *softc);
+int bnxt_create_trusted_vf_sysctls(struct bnxt_softc *softc, uint16_t num_vfs);
+int bnxt_create_spoofchk_vf_sysctls(struct bnxt_softc *softc, uint16_t num_vfs);
+bool bnxt_is_trusted_vf(struct bnxt_softc *bp, struct bnxt_vf_info *vf);
+void bnxt_hwrm_exec_fwd_req(struct bnxt_softc *bp);
+void bnxt_destroy_trusted_vf_sysctls(struct bnxt_softc *softc);
+int bnxt_set_vf_trust(struct bnxt_softc *softc, int vf_id, bool trusted);
+int bnxt_approve_mac(struct bnxt_softc *sc);
+void bnxt_update_vf_mac(struct bnxt_softc *sc);
+bool bnxt_promisc_ok(struct bnxt_softc *softc);
+int bnxt_set_vf_spoofchk(struct bnxt_softc *sc, int vf_id, bool enable);
+int bnxt_cfg_hw_sriov(struct bnxt_softc *softc, uint16_t *num_vfs, bool reset);
+void bnxt_reenable_sriov(struct bnxt_softc *bp);
+
+
+#endif /* _BNXT_SRIOV_H_ */

sys/dev/bnxt/bnxt_en/if_bnxt.c

@@ -266,6 +266,11 @@ static device_method_t bnxt_methods[] = {
 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
 	DEVMETHOD(device_suspend, iflib_device_suspend),
 	DEVMETHOD(device_resume, iflib_device_resume),
+#ifdef PCI_IOV
+	DEVMETHOD(pci_iov_init, iflib_device_iov_init),
+	DEVMETHOD(pci_iov_uninit, iflib_device_iov_uninit),
+	DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf),
+#endif
 	DEVMETHOD_END
 };
 
@@ -344,7 +349,11 @@ static device_method_t bnxt_iflib_methods[] = {
 	DEVMETHOD(ifdi_i2c_req, bnxt_i2c_req),
 
 	DEVMETHOD(ifdi_needs_restart, bnxt_if_needs_restart),
-
+#ifdef PCI_IOV
+	DEVMETHOD(ifdi_iov_init, bnxt_iov_init),
+	DEVMETHOD(ifdi_iov_uninit, bnxt_iov_uninit),
+	DEVMETHOD(ifdi_iov_vf_add, bnxt_iov_vf_add),
+#endif
 	DEVMETHOD_END
 };
 
@@ -2738,6 +2747,12 @@ bnxt_attach_post(if_ctx_t ctx)
 	bnxt_dcb_init(softc);
 	bnxt_rdma_aux_device_init(softc);
 
+#if PCI_IOV
+	/* SR-IOV attach */
+	if (BNXT_PF(softc) && BNXT_CHIP_P5_PLUS(softc))
+		bnxt_sriov_attach(softc);
+#endif
+
 failed:
 	return rc;
 }
@@ -5324,4 +5339,4 @@ bnxt_get_wol_settings(struct bnxt_softc *softc)
 	do {
 		wol_handle = bnxt_hwrm_get_wol_fltrs(softc, wol_handle);
 	} while (wol_handle && wol_handle != BNXT_NO_MORE_WOL_FILTERS);
-}
+}

sys/modules/bnxt/bnxt_en/Makefile

@@ -9,6 +9,7 @@ SRCS	+= bnxt_sysctl.c
 SRCS	+= bnxt_mgmt.c
 SRCS	+= bnxt_dcb.c bnxt_dcb.h
 SRCS	+= bnxt_auxbus_compat.c bnxt_auxbus_compat.h
+SRCS	+= bnxt_sriov.c bnxt_sriov.h
 SRCS	+= bnxt_ulp.c bnxt_ulp.h
 SRCS	+= ${LINUXKPI_GENSRCS}