if_bnxt: add MPC (Mid-Path Channel) ring infrastructure

Add MPC ring support: a pair of extra, driver-private
TX/completion/notification rings the firmware uses as a side-channel
for offload commands. The only current consumer is kTLS TX crypto
command submission; RCE/CFA channel types are not yet implemented by
any consumer.

Adds MPC ring allocation/teardown, HWRM ring alloc/free, a private
interrupt path for the MPC NQ ring, and the TX submit/completion
path. The RoCE-only IRQ table builder is split into a generic,
exported bnxt_populate_irq(softc, irq_count) that both MPC and RoCE
use, plus an incremental bnxt_populate_irq_roce() that grows the
table on top of whatever is already there, so IRQ rids stay
consistent regardless of attach order.

This patch does not link standalone: bnxt_mpc_cmp() calls into
kTLS's completion handler, added by a later patch.

Reviewed by:    gallatin
MFC after:      2 weeks
Sponsored by:   Broadcom Inc.
Differential Revision:  https://reviews.freebsd.org/D58603

sys/conf/files

@@ -1333,6 +1333,7 @@ dev/bnxt/bnxt_en/bnxt_log.c		optional bnxt iflib pci compile-with "${BNXT_C}"
 dev/bnxt/bnxt_en/bnxt_log_data.c	optional bnxt iflib pci compile-with "${BNXT_C}"
 dev/bnxt/bnxt_en/bnxt_coredump.c	optional bnxt iflib pci compile-with "${BNXT_C}"
 dev/bnxt/bnxt_en/bnxt_ptp.c		optional bnxt iflib pci compile-with "${BNXT_C}"
+dev/bnxt/bnxt_en/bnxt_mpc.c		optional bnxt iflib pci compile-with "${BNXT_C}"
 dev/bnxt/bnxt_en/if_bnxt.c		optional bnxt iflib pci compile-with "${BNXT_C}"
 dev/bwi/bwimac.c		optional bwi
 dev/bwi/bwiphy.c		optional bwi

sys/dev/bnxt/bnxt_en/bnxt.h

@@ -608,6 +608,11 @@ struct tx_bd_opaque {
 
 #define TX_RING_MASK(txr) ((txr)->ring_size - 1)
 
+struct bnxt_sw_mpc_tx_bd {
+	uint8_t inline_bds;
+	unsigned long handle;
+};
+
 struct bnxt_ring {
 	uint64_t		paddr;
 	vm_offset_t		doorbell;
@@ -618,6 +623,15 @@ struct bnxt_ring {
 	uint16_t		phys_id;
 	uint16_t		idx;
 	uint16_t		running_bds;	/* host TX-completion-coalescing accumulator */
+	uint8_t			queue_id;	/* CoS queue, or BNXT_MPC_QUEUE_ID below */
+#define BNXT_MPC_QUEUE_ID	0xff
+	uint8_t			mpc_chnl_type;	/* valid when queue_id == BNXT_MPC_QUEUE_ID */
+	struct iflib_dma_info	ring_mem;	/* manual DMA alloc for driver-private rings (MPC) */
+	struct bnxt_cp_ring	*cp_ring;	/* completion ring this TX ring reports to (MPC) */
+	uint16_t		prod;		/* MPC TX ring producer index */
+	uint16_t		cons;		/* MPC TX ring consumer index */
+	struct mtx		tx_lock;	/* MPC TX ring lock */
+	struct bnxt_sw_mpc_tx_bd *tx_mpc_buf_ring;
 	struct bnxt_full_tpa_start *tpa_start;
 	union {
 		u64             db_key64;
@@ -649,6 +663,10 @@ struct bnxt_cp_ring {
 	uint8_t			type;
 #define Q_TYPE_TX		1
 #define Q_TYPE_RX		2
+#define TX_CP_NQ		3	/* MPC-private NQ ring (bnxt_mpc.c) */
+	int			msix_vec;	/* MPC-private ring: bus_setup_intr vector */
+	struct resource		*irq_res;	/* MPC-private ring: bus_setup_intr resource */
+	void			*irq_cookie;	/* MPC-private ring: bus_setup_intr cookie */
 };
 
 struct bnxt_full_tpa_start {
@@ -718,6 +736,10 @@ struct bnxt_func_qcfg {
 	uint16_t alloc_completion_rings;
 	uint16_t alloc_tx_rings;
 	uint16_t alloc_rx_rings;
+	/* Firmware-granted counts before the MPC reservation is subtracted;
+	 * bnxt_set_dflt_mpc_rings() needs these unreduced values. */
+	uint16_t orig_alloc_completion_rings;
+	uint16_t orig_alloc_tx_rings;
 	uint16_t alloc_vnics;
 	uint16_t alloc_rss_ctx;
 	uint16_t alloc_l2_ctx;
@@ -1435,6 +1457,8 @@ struct bnxt_softc {
 	struct bnxt_ptp_cfg	*ptp_cfg;
 	bool			rx_ts_enabled;
 
+	struct bnxt_mpc_info	*mpc_info;
+
 #define BNXT_CAGR_CQCOAL_OFFSET			0xc00
 #define BNXT_CAGR_NQAGG_MAXTIMER		0x5930000
 #define BNXT_CAGR_TICK_RES_DEFAULT		0x2
@@ -1523,6 +1547,10 @@ int bnxt_alloc_ctx_pg_tbls(struct bnxt_softc *softc,
 			    uint32_t mem_size, uint8_t depth,
 			    struct bnxt_ctx_mem_type *ctxm);
 int bnxt_hwrm_ptp_qcfg(struct bnxt_softc *bp);
+int bnxt_populate_irq(struct bnxt_softc *softc, int irq_count);
+void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr);
+void bnxt_set_db_mask(struct bnxt_softc *bp, struct bnxt_ring *db,
+    uint32_t ring_type);
 
 static inline u32
 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx)

sys/dev/bnxt/bnxt_en/bnxt_hwrm.c

@@ -32,6 +32,7 @@
 
 #include "bnxt.h"
 #include "bnxt_hwrm.h"
+#include "bnxt_mpc.h"
 #include "hsi_struct_def.h"
 #include "bnxt_coredump.h"
 
@@ -1205,6 +1206,7 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
 	    (void *)softc->hwrm_cmd_resp.idi_vaddr;
 	struct bnxt_func_info *func = &softc->func;
 	uint32_t flags, flags_ext, flags_ext2;
+	uint8_t mpc_chnls_cap = 0;
 
 	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCAPS);
 	req.fid = htole16(0xffff);
@@ -1354,8 +1356,18 @@ bnxt_hwrm_func_qcaps(struct bnxt_softc *softc)
 		}
 	}
 
+	/*
+	 * Only extract the field needed here; the actual alloc/free work
+	 * (malloc(M_NOWAIT)/free() in bnxt_alloc_mpc_info()) happens after
+	 * BNXT_HWRM_UNLOCK() below instead of under hwrm_lock.
+	 */
+	if (BNXT_PF(softc))
+		mpc_chnls_cap = resp->mpc_chnls_cap;
+
 fail:
 	BNXT_HWRM_UNLOCK(softc);
+	if (!rc && BNXT_PF(softc))
+		bnxt_alloc_mpc_info(softc, mpc_chnls_cap);
 	return rc;
 }
 
@@ -1468,6 +1480,22 @@ bnxt_hwrm_func_qcfg(struct bnxt_softc *softc)
 	fn_qcfg->alloc_stat_ctx = le16toh(resp->alloc_stat_ctx);
 	fn_qcfg->alloc_msix = le16toh(resp->alloc_msix);
 
+	fn_qcfg->orig_alloc_completion_rings = fn_qcfg->alloc_completion_rings;
+	fn_qcfg->orig_alloc_tx_rings = fn_qcfg->alloc_tx_rings;
+
+	/* Reserve minimum rings for MPC */
+	if (BNXT_PF(softc)) {
+		if (softc->mpc_info) {
+			uint16_t mpc_min = BNXT_MIN_MPC_TCE + BNXT_MIN_MPC_RCE;
+
+			/* Clamp to 0 instead of underflowing these unsigned counts. */
+			fn_qcfg->alloc_completion_rings -=
+			    min(fn_qcfg->alloc_completion_rings, mpc_min);
+			fn_qcfg->alloc_tx_rings -=
+			    min(fn_qcfg->alloc_tx_rings, mpc_min);
+		}
+	}
+
 	switch (resp->port_partition_type) {
 	case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0:
 	case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_2:
@@ -1976,17 +2004,24 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type,
 
 	switch (type) {
 	case HWRM_RING_ALLOC_INPUT_RING_TYPE_TX:
-		cp_ring = &softc->tx_cp_rings[idx];
+		if (ring->queue_id == BNXT_MPC_QUEUE_ID) {
+			cp_ring = ring->cp_ring;
+			req.mpc_chnls_type = ring->mpc_chnl_type;
+			req.enables |= htole32(HWRM_RING_ALLOC_INPUT_ENABLES_MPC_CHNLS_TYPE);
+		} else {
+			cp_ring = &softc->tx_cp_rings[idx];
 
-		req.cmpl_ring_id = htole16(cp_ring->ring.phys_id);
-		/* queue_id - what CoS queue the TX ring is associated with */
-		req.queue_id = htole16(softc->tx_q_info[0].queue_id);
+			/* queue_id - what CoS queue the TX ring is associated with */
+			req.queue_id = htole16(softc->tx_q_info[0].queue_id);
 
+			req.enables |= htole32(
+			    HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID);
+		}
 		req.stat_ctx_id = htole32(cp_ring->stats_ctx_id);
-		req.enables |= htole32(
-		    HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID);
-		if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL)
+		if (softc->fw_cap & BNXT_FLAG_TX_COAL_CMPL &&
+		    ring->queue_id != BNXT_MPC_QUEUE_ID)
 			req.cmpl_coal_cnt = softc->tx_hw_coal_cnt;
+		req.cmpl_ring_id = htole16(cp_ring->ring.phys_id);
 		break;
 	case HWRM_RING_ALLOC_INPUT_RING_TYPE_RX:
 		if (!BNXT_CHIP_P5_PLUS(softc))
@@ -2023,7 +2058,10 @@ bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type,
 		}
 
                 req.cq_handle = htole64(ring->id);
-		req.nq_ring_id = htole16(softc->nq_rings[idx].ring.phys_id);
+		if (ring->queue_id == BNXT_MPC_QUEUE_ID)
+			req.nq_ring_id = htole16(softc->mpc_info->mpc_nq_rings[idx].ring.phys_id);
+		else
+			req.nq_ring_id = htole16(softc->nq_rings[idx].ring.phys_id);
 		req.enables |= htole32(
 			HWRM_RING_ALLOC_INPUT_ENABLES_NQ_RING_ID_VALID);
 		break;

sys/dev/bnxt/bnxt_en/bnxt_mpc.h

@@ -0,0 +1,182 @@
+/*
+ * Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2026 Broadcom, All Rights Reserved.
+ * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef BNXT_MPC_H
+#define BNXT_MPC_H
+
+#define BNXT_MPC_TCE_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_TCE
+#define BNXT_MPC_RCE_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_RCE
+#define BNXT_MPC_TE_CFA_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_TE_CFA
+#define BNXT_MPC_RE_CFA_TYPE HWRM_RING_ALLOC_INPUT_MPC_CHNLS_TYPE_RE_CFA
+#define BNXT_MPC_TYPE_MAX (BNXT_MPC_RE_CFA_TYPE + 1)
+
+#define BNXT_MAX_MPC 8
+
+#define BNXT_MIN_MPC_TCE 1
+#define BNXT_MIN_MPC_RCE 1
+#define BNXT_DFLT_MPC_TCE BNXT_MAX_MPC
+#define BNXT_DFLT_MPC_RCE BNXT_MAX_MPC
+
+#define BNXT_MIN_MPC_TE_CFA 1
+#define BNXT_MIN_MPC_RE_CFA 1
+#define BNXT_DFLT_MPC_TE_CFA BNXT_MAX_MPC
+#define BNXT_DFLT_MPC_RE_CFA BNXT_MAX_MPC
+
+#define BNXT_MPC_TMO_USECS (1000 * 1000)
+
+struct bnxt_mpc_info {
+	uint8_t mpc_chnls_cap;
+	uint8_t mpc_cp_rings_count;
+	uint8_t mpc_ring_count[BNXT_MPC_TYPE_MAX];
+	struct bnxt_ring *mpc_rings[BNXT_MPC_TYPE_MAX];
+	struct bnxt_cp_ring *mpc_cp_rings;
+	struct bnxt_cp_ring *mpc_nq_rings;
+	struct iflib_dma_info   tx_stats[BNXT_MAX_NUM_QUEUES];
+};
+
+#define BNXT_RING_MPC_IDX_TYPE_MASK 0xf0
+#define BNXT_RING_MPC_IDX_TYPE_SFT 4
+#define BNXT_RING_MPC_IDX_RING_MASK 0x0f
+
+enum bnxt_mpc_type {
+	BNXT_MPC_CRYPTO,
+	BNXT_MPC_CFA,
+};
+
+#define BNXT_MPC_CRYPTO_CAP                                                    \
+	(HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_TCE |                            \
+	 HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_RCE)
+
+#define BNXT_MPC_CRYPTO_CAPABLE(softc)                                         \
+	((softc)->mpc_info ? ((softc)->mpc_info->mpc_chnls_cap &               \
+			      BNXT_MPC_CRYPTO_CAP) == BNXT_MPC_CRYPTO_CAP      \
+			   : false)
+
+#define BNXT_MPC_CFA_CAP                                                       \
+	(HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_TE_CFA |                         \
+	 HWRM_FUNC_QCAPS_OUTPUT_MPC_CHNLS_CAP_RE_CFA)
+
+#define BNXT_MPC_CFA_CAPABLE(bp)                                               \
+	((bp)->mpc_info ? ((bp)->mpc_info->mpc_chnls_cap &                     \
+			   BNXT_MPC_CFA_CAP) == BNXT_MPC_CFA_CAP               \
+			: false)
+
+#define MPC_CMP_RING_MASK(cpr) ((cpr)->ring.ring_size - 1)
+#define MPC_RING_CMP(cpr, idx) ((idx) & MPC_CMP_RING_MASK(cpr))
+
+#define RING_TX(txr, idx) ((idx) & TX_RING_MASK(txr))
+#define NEXT_TX(txr, idx) ((idx + 1) & TX_RING_MASK(txr))
+#define ADVANCE_TX_BY(txr, idx, count) ((idx + count) & TX_RING_MASK(txr))
+
+#define TX_OPAQUE_IDX_MASK 0x0000ffff
+#define TX_OPAQUE_BDS_MASK 0x00ff0000
+#define TX_OPAQUE_BDS_SHIFT 16
+#define TX_OPAQUE_RING_MASK 0xff000000
+#define TX_OPAQUE_RING_SHIFT 24
+
+#define SET_TX_OPAQUE(txr, prod, bds)                                          \
+	(((txr)->idx << TX_OPAQUE_RING_SHIFT) |                                \
+	 ((bds) << TX_OPAQUE_BDS_SHIFT) |                                      \
+	 (((prod) & TX_RING_MASK(txr)) & TX_OPAQUE_IDX_MASK))
+
+#define TX_OPAQUE_IDX(opq) ((opq) & TX_OPAQUE_IDX_MASK)
+#define TX_OPAQUE_RING(opq)                                                    \
+	(((opq) & TX_OPAQUE_RING_MASK) >> TX_OPAQUE_RING_SHIFT)
+#define TX_OPAQUE_BDS(opq) (((opq) & TX_OPAQUE_BDS_MASK) >> TX_OPAQUE_BDS_SHIFT)
+
+#define BNXT_INV_MPC_HDL (-1UL)
+
+struct bnxt_cmpl_entry {
+	void *cmpl;
+	uint32_t len;
+};
+
+struct mpc_cmp {
+	__le32 mpc_cmp_client_subtype_type;
+#define MPC_CMP_TYPE (0x3f << 0)
+#define MPC_CMP_TYPE_MID_PATH_SHORT 0x1e
+#define MPC_CMP_TYPE_MID_PATH_LONG 0x1f
+
+#define MPC_CMP_SUBTYPE 0xf00
+#define MPC_CMP_SUBTYPE_SFT 8
+#define MPC_CMP_SUBTYPE_SOLICITED (0x0 << 8)
+#define MPC_CMP_SUBTYPE_ERR (0x1 << 8)
+#define MPC_CMP_SUBTYPE_RESYNC (0x2 << 8)
+
+#define MPC_CMP_CLIENT (0xf << 12)
+#define MPC_CMP_CLIENT_SFT 12
+#define MPC_CMP_CLIENT_TCE (0x0 << 12)
+#define MPC_CMP_CLIENT_RCE (0x1 << 12)
+#define MPC_CMP_CLIENT_TE_CFA (0x2 << 12)
+#define MPC_CMP_CLIENT_RE_CFA (0x3 << 12)
+
+	uint32_t mpc_cmp_opaque;
+	__le32 mpc_cmp_v;
+#define MPC_CMP_V (1 << 0)
+	__le32 mpc_cmp_filler;
+};
+
+#define MPC_CMP_CMP_TYPE(mpcmp)                                                \
+	(le32toh((mpcmp)->mpc_cmp_client_subtype_type) & MPC_CMP_TYPE)
+
+#define MPC_CMP_CLIENT_TYPE(mpcmp)                                             \
+	(le32toh((mpcmp)->mpc_cmp_client_subtype_type) & MPC_CMP_CLIENT)
+
+/*
+ * ERR and RESYNC are both unsolicited (firmware-initiated, no matching TX
+ * descriptor); only SOLICITED completions carry a valid TX opaque to parse.
+ */
+#define MPC_CMP_UNSOLICIT_SUBTYPE(mpcmp)                                       \
+	(((le32toh((mpcmp)->mpc_cmp_client_subtype_type) &                     \
+	   MPC_CMP_SUBTYPE) == MPC_CMP_SUBTYPE_ERR) ||                         \
+	 ((le32toh((mpcmp)->mpc_cmp_client_subtype_type) &                     \
+	   MPC_CMP_SUBTYPE) == MPC_CMP_SUBTYPE_RESYNC))
+
+#define MPC_CMP_VALID(cpr, mpcmp, raw_cons)                                    \
+	(!!((mpcmp)->mpc_cmp_v & htole32(MPC_CMP_V)) ==                    \
+	 !((raw_cons) & (cpr)->ring.ring_size))
+
+void bnxt_alloc_mpc_info(struct bnxt_softc *softc, uint8_t mpc_chnls_cap);
+void bnxt_free_mpc_info(struct bnxt_softc *softc);
+void bnxt_set_dflt_mpc_rings(struct bnxt_softc *softc);
+int bnxt_alloc_mpcs(struct bnxt_softc *softc);
+void bnxt_free_mpcs(struct bnxt_softc *softc);
+int bnxt_alloc_mpc_rings(struct bnxt_softc *softc);
+void bnxt_free_mpc_rings(struct bnxt_softc *softc);
+int bnxt_mpc_irq_setup(struct bnxt_softc *softc);
+void bnxt_mpc_irq_cleanup(struct bnxt_softc *softc);
+int bnxt_hwrm_mpc_ring_alloc(struct bnxt_softc *softc);
+int bnxt_hwrm_mpc_ring_free(struct bnxt_softc *softc);
+int bnxt_start_xmit_mpc(struct bnxt_softc *softc, struct bnxt_ring *txr,
+			void *data, uint len, unsigned long handle);
+int bnxt_mpc_cmp(struct bnxt_softc *softc, struct bnxt_cp_ring *cpr,
+		 uint32_t *raw_cons);
+void bnxt_clear_mpc_rings_ids(struct bnxt_softc *softc);
+struct bnxt_ring *bnxt_select_mpc_ring(struct bnxt_mpc_info *mpc);
+
+#endif

sys/dev/bnxt/bnxt_en/bnxt_ulp.c

@@ -426,7 +426,11 @@ void bnxt_destroy_irq(struct bnxt_softc *softc)
 	kfree(softc->irq_tbl);
 }
 
-static int bnxt_populate_irq(struct bnxt_softc *softc)
+/*
+ * Build the IRQ table from scratch for isc_ntxqsets + irq_count vectors;
+ * usable by any feature needing private IRQs without prior setup.
+ */
+int bnxt_populate_irq(struct bnxt_softc *softc, int irq_count)
 {
 	struct resource_list *rl = NULL;
 	struct resource_list_entry *rle = NULL;
@@ -434,16 +438,25 @@ static int bnxt_populate_irq(struct bnxt_softc *softc)
 	struct pci_devinfo *dinfo = NULL;
 	int i;
 
-	softc->total_irqs = softc->scctx->isc_nrxqsets + BNXT_ROCE_IRQ_COUNT;
+	softc->total_irqs = softc->scctx->isc_ntxqsets + irq_count;
 	irq_tbl = kzalloc(softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL);
 
 	if (!irq_tbl) {
 		device_printf(softc->dev, "Failed to allocate IRQ table\n");
 		return -1;
 	}
+
 	dinfo = device_get_ivars(softc->pdev->dev.bsddev);
 	rl = &dinfo->resources;
 	rle = resource_list_find(rl, SYS_RES_IRQ, 1);
+	if (rle == NULL) {
+		device_printf(softc->dev,
+		    "No default resources for rid = %d, type = %d\n",
+		    1, SYS_RES_IRQ);
+		kfree(irq_tbl);
+		return -1;
+	}
+
 	softc->pdev->dev.irq_start = rle->start;
 	softc->pdev->dev.irq_end = rle->start + softc->total_irqs;
 
@@ -457,6 +470,56 @@ static int bnxt_populate_irq(struct bnxt_softc *softc)
 	return 0;
 }
 
+/*
+ * Grow the IRQ table by BNXT_ROCE_IRQ_COUNT on top of existing entries (e.g.
+ * MPC's) instead of reassigning total_irqs, to avoid reclaiming MPC's rids.
+ */
+static int bnxt_populate_irq_roce(struct bnxt_softc *softc)
+{
+	struct resource_list *rl = NULL;
+	struct resource_list_entry *rle = NULL;
+	struct pci_devinfo *dinfo = NULL;
+	struct bnxt_msix_tbl *irq_tbl;
+	int prev_total_irqs = softc->total_irqs;
+	int i;
+
+	softc->total_irqs += BNXT_ROCE_IRQ_COUNT;
+	irq_tbl = krealloc(softc->irq_tbl,
+	    softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL);
+
+	if (!irq_tbl) {
+		device_printf(softc->dev, "Failed to allocate IRQ table\n");
+		/* Roll back total_irqs since the table itself wasn't grown. */
+		softc->total_irqs = prev_total_irqs;
+		return -1;
+	}
+
+	dinfo = device_get_ivars(softc->pdev->dev.bsddev);
+	rl = &dinfo->resources;
+	rle = resource_list_find(rl, SYS_RES_IRQ, 1);
+	if (rle == NULL) {
+		device_printf(softc->dev,
+		    "No default resources for rid = %d, type = %d\n",
+		    1, SYS_RES_IRQ);
+		/* irq_tbl isn't installed yet; free it and roll back total_irqs. */
+		kfree(irq_tbl);
+		softc->total_irqs = prev_total_irqs;
+		return -1;
+	}
+
+	softc->pdev->dev.irq_start = rle->start;
+	softc->pdev->dev.irq_end = rle->start + softc->total_irqs;
+
+	for (i = prev_total_irqs; i < softc->total_irqs; i++) {
+		irq_tbl[i].entry = i;
+		irq_tbl[i].vector = softc->pdev->dev.irq_start + i;
+	}
+
+	softc->irq_tbl = irq_tbl;
+
+	return 0;
+}
+
 static const struct bnxt_en_ops bnxt_en_ops_tbl = {
 	.bnxt_register_device	= bnxt_register_dev,
 	.bnxt_unregister_device	= bnxt_unregister_dev,
@@ -521,7 +584,7 @@ int bnxt_rdma_aux_device_add(struct bnxt_softc *bp)
 	struct auxiliary_device *aux_dev;
 	int ret = -1;
 
-	if (bnxt_populate_irq(bp))
+	if (bnxt_populate_irq_roce(bp))
 		return ret;
 
 	device_printf(bp->dev, "V:D:SV:SD %x:%x:%x:%x, irq 0x%x, "

sys/dev/bnxt/bnxt_en/if_bnxt.c

@@ -76,6 +76,7 @@
 #include "bnxt_log_data.h"
 #include "bnxt_coredump.h"
 #include "bnxt_ptp.h"
+#include "bnxt_mpc.h"
 
 /*
  * PCI Device ID Table
@@ -260,7 +261,7 @@ static int bnxt_handle_isr(void *arg);
 static void bnxt_clear_ids(struct bnxt_softc *softc);
 static void inline bnxt_do_enable_intr(struct bnxt_cp_ring *cpr);
 static void inline bnxt_do_disable_intr(struct bnxt_cp_ring *cpr);
-static void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr);
+void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr);
 static void bnxt_def_cp_task(void *context, int pending);
 static void bnxt_handle_async_event(struct bnxt_softc *softc,
     struct cmpl_base *cmpl);
@@ -449,6 +450,8 @@ bnxt_init_sctx_variants(uint16_t device_id)
 	bnxt_sctx_pf_init = bnxt_sctx_template;
 
     bnxt_sctx_pf_init.isc_admin_intrcnt = BNXT_ROCE_IRQ_COUNT;
+    if (device_id == BCM57608)
+	bnxt_sctx_pf_init.isc_admin_intrcnt += BNXT_MAX_MPC;
 
     bnxt_sctx_vf_init = bnxt_sctx_template;
     bnxt_sctx_vf_init.isc_flags |= IFLIB_IS_VF;
@@ -553,7 +556,7 @@ bnxt_nq_free(struct bnxt_softc *softc)
 }
 
 
-static void
+void
 bnxt_set_db_mask(struct bnxt_softc *bp, struct bnxt_ring *db,
 		 u32 ring_type)
 {
@@ -668,6 +671,15 @@ bnxt_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs,
 	}
 
 	softc->ntxqsets = ntxqsets;
+	bnxt_set_dflt_mpc_rings(softc);
+	rc = bnxt_alloc_mpcs(softc);
+	if (rc == 0)
+		rc = bnxt_alloc_mpc_rings(softc);
+	if (rc) {
+		bnxt_free_mpc_rings(softc);
+		bnxt_free_mpcs(softc);
+		goto dma_alloc_fail;
+	}
 	return rc;
 
 dma_alloc_fail:
@@ -709,6 +721,9 @@ bnxt_queues_free(if_ctx_t ctx)
 	free(softc->rx_rings, M_DEVBUF);
 	free(softc->rx_cp_rings, M_DEVBUF);
 	bnxt_nq_free(softc);
+
+	bnxt_free_mpc_rings(softc);
+	bnxt_free_mpcs(softc);
 }
 
 static int
@@ -1296,12 +1311,21 @@ bnxt_bs_trace_init(struct bnxt_softc *bp, struct bnxt_ctx_mem_type *ctxm)
 static int
 bnxt_backing_store_cfg_v2(struct bnxt_softc *softc, u32 ena)
 {
+	struct bnxt_mpc_info *mpc = softc->mpc_info;
 	struct bnxt_ctx_mem_info *ctx = softc->ctx_mem;
 	struct bnxt_ctx_mem_type *ctxm;
 	u16 last_type = BNXT_CTX_INV;
 	int rc = 0;
 	u16 type;
 
+	if (BNXT_PF(softc) && mpc && mpc->mpc_chnls_cap) {
+		ctxm = &ctx->ctx_arr[BNXT_CTX_MTQM];
+		rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, ctxm->max_entries, 1);
+		if (rc)
+			return rc;
+		last_type = BNXT_CTX_MTQM;
+	}
+
 	if (BNXT_PF(softc)) {
 		for (type = BNXT_CTX_SRT_TRACE; type <= BNXT_CTX_ROCE_HWRM_TRACE; type++) {
 			ctxm = &ctx->ctx_arr[type];
@@ -2971,6 +2995,7 @@ bnxt_attach_pre(if_ctx_t ctx)
 
 failed:
 	bnxt_free_sysctl_ctx(softc);
+	bnxt_free_mpc_info(softc);
 init_sysctl_failed:
 	bnxt_hwrm_func_drv_unrgtr(softc, false);
 	if (BNXT_PF(softc))
@@ -3022,6 +3047,10 @@ bnxt_attach_post(if_ctx_t ctx)
 
 	softc->rx_buf_size = min(softc->scctx->isc_max_frame_size, BNXT_PAGE_SIZE);
 	bnxt_dcb_init(softc);
+	rc = bnxt_mpc_irq_setup(softc);
+	if (rc)
+		goto failed;
+
 	bnxt_rdma_aux_device_init(softc);
 
 	/* SR-IOV attach */
@@ -3034,6 +3063,8 @@ bnxt_attach_post(if_ctx_t ctx)
 		    "crash dump init failure rc: %d\n", rc);
 
 failed:
+	/* No cleanup needed: on nonzero return iflib_device_register() calls
+	 * IFDI_DETACH and IFDI_QUEUES_FREE, which tear down anything set up so far. */
 	return rc;
 }
 
@@ -3045,6 +3076,7 @@ bnxt_detach(if_ctx_t ctx)
 	struct bnxt_vlan_tag *tmp;
 	int i;
 
+	bnxt_mpc_irq_cleanup(softc);
 	bnxt_rdma_aux_device_uninit(softc);
 	cancel_delayed_work_sync(&softc->fw_reset_task);
 	cancel_work_sync(&softc->sp_task);
@@ -3057,6 +3089,11 @@ bnxt_detach(if_ctx_t ctx)
 	bnxt_free_sysctl_ctx(softc);
 	bnxt_free_crash_dump_mem(softc);
 	bnxt_hwrm_func_reset(softc);
+	/* Must run before bnxt_free_mpc_info(): these gate on mpc_info being
+	 * non-NULL, so freeing it first would leak the MPC rings. */
+	bnxt_free_mpc_rings(softc);
+	bnxt_free_mpcs(softc);
+	bnxt_free_mpc_info(softc);
 	bnxt_free_ctx_mem(softc);
 	bnxt_clear_ids(softc);
 	iflib_irq_free(ctx, &softc->def_cp_ring.irq);
@@ -3133,6 +3170,11 @@ bnxt_hwrm_resource_free(struct bnxt_softc *softc)
 		if (rc)
 			goto fail;
 	}
+
+	rc = bnxt_hwrm_mpc_ring_free(softc);
+	if (rc)
+		goto fail;
+
 	rc = bnxt_hwrm_free_filter(softc);
 	if (rc)
 		goto fail;
@@ -3520,6 +3562,11 @@ skip_def_cp_ring:
 		softc->db_ops.bnxt_db_tx(&softc->tx_rings[i], 0);
 	}
 
+	if (BNXT_PF(softc)) {
+		rc = bnxt_hwrm_mpc_ring_alloc(softc);
+		if (rc)
+			goto fail;
+	}
 	bnxt_do_enable_intr(&softc->def_cp_ring);
 	bnxt_get_port_module_status(softc);
 	bnxt_media_status(softc->ctx, &ifmr);
@@ -5303,6 +5350,7 @@ bnxt_clear_ids(struct bnxt_softc *softc)
 		softc->ag_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE;
 		softc->grp_info[i].grp_id = (uint16_t)HWRM_NA_SIGNATURE;
 	}
+	bnxt_clear_mpc_rings_ids(softc);
 	softc->vnic_info.filter_id = -1;
 	softc->vnic_info.id = (uint16_t)HWRM_NA_SIGNATURE;
 	softc->vnic_info.rss_id = (uint16_t)HWRM_NA_SIGNATURE;
@@ -5310,7 +5358,7 @@ bnxt_clear_ids(struct bnxt_softc *softc)
 	    softc->vnic_info.rss_grp_tbl.idi_size);
 }
 
-static void
+void
 bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr)
 {
 	struct cmpl_base *cmp = (void *)cpr->ring.vaddr;

sys/modules/bnxt/bnxt_en/Makefile

@@ -15,6 +15,7 @@ SRCS	+= bnxt_log.c bnxt_log.h
 SRCS	+= bnxt_log_data.c bnxt_log_data.h
 SRCS	+= bnxt_coredump.c bnxt_coredump.h
 SRCS	+= bnxt_ptp.c bnxt_ptp.h
+SRCS	+= bnxt_mpc.c bnxt_mpc.h
 SRCS	+= ${LINUXKPI_GENSRCS}
 
 CFLAGS+= -DIFLIB