igbv: Retry transient VLAN admission failures

A PF mailbox NACK does not distinguish the SR-IOV VLAN request rate
limit from permanent VLVF exhaustion. Preserve desired VLAN membership
and retry four additions per 500 ms timer tick, matching the PF
sustained allowance.

Bound the whole recovery batch to eight seconds from its first failure
and consolidate restore diagnostics, so a full table cannot create a
permanent mailbox poller or repeated per-VID log bursts.

Sponsored by:	BBOX.io

sys/dev/e1000/if_em.c

@@ -4581,9 +4581,11 @@ em_if_vlan_register(if_ctx_t ctx, u16 vtag)
 		++sc->num_vlans;
 	if (sc->vf_ifp &&
 	    e1000_vfta_set_vf(&sc->hw, vtag, true) != E1000_SUCCESS) {
+		igbv_vlan_retry_add(sc, vtag);
 		device_printf(sc->dev,
 		    "VF VLAN %u add request failed\n", vtag);
-	}
+	} else if (sc->vf_ifp)
+		igbv_vlan_retry_clear(sc, vtag);
 	if (!sc->vf_ifp) {
 		if (igb_iov_enabled(sc))
 			igb_iov_rebuild_vlan(sc);
@@ -4602,6 +4604,8 @@ em_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
 	index = (vtag >> 5) & 0x7F;
 	mask = 1U << (vtag & 0x1F);
 	present = (sc->shadow_vfta[index] & mask) != 0;
+	if (sc->vf_ifp)
+		igbv_vlan_retry_clear(sc, vtag);
 	if (sc->vf_ifp &&
 	    e1000_vfta_set_vf(&sc->hw, vtag, false) != E1000_SUCCESS) {
 		device_printf(sc->dev,
@@ -4704,6 +4708,7 @@ em_setup_vlan_hw_support(if_ctx_t ctx)
 	s32 error;
 	u32 max_frame_size, reg;
 	u16 vid;
+	int restore_failures;
 
 	/*
 	 * Only PFs have control over VLAN HW filtering
@@ -4714,6 +4719,7 @@ em_setup_vlan_hw_support(if_ctx_t ctx)
 		max_frame_size = min(sc->shared->isc_max_frame_size +
 		    VLAN_TAG_SIZE, IGB_IOV_MAX_FRAME_SIZE);
 		e1000_rlpml_set_vf(hw, max_frame_size);
+		restore_failures = 0;
 		for (vid = 0; vid < 4096; vid++) {
 			if ((sc->shadow_vfta[vid >> 5] &
 			    (1U << (vid & 0x1f))) == 0)
@@ -4723,11 +4729,16 @@ em_setup_vlan_hw_support(if_ctx_t ctx)
 			 * replay if the PF mailbox is absent during reset.
 			 */
 			error = e1000_vfta_set_vf(hw, vid, true);
-			if (error != E1000_SUCCESS)
-				device_printf(sc->dev,
-				    "VF VLAN %u restore request failed\n",
-				    vid);
+			if (error != E1000_SUCCESS) {
+				igbv_vlan_retry_add(sc, vid);
+				restore_failures++;
+			} else
+				igbv_vlan_retry_clear(sc, vid);
 		}
+		if (restore_failures != 0)
+			device_printf(sc->dev,
+			    "VF VLAN restore failed for %d VIDs; retrying\n",
+			    restore_failures);
 		return;
 	}
 

sys/dev/e1000/if_em.h

@@ -584,6 +584,9 @@ struct e1000_softc {
 	*/
 	u32			shadow_vfta[EM_VFTA_SIZE];
 	u32			vf_vfta_stale[EM_VFTA_SIZE];
+	u32			vf_vfta_retry[EM_VFTA_SIZE];
+	sbintime_t		vf_vlan_retry_deadline;
+	u16			vf_vlan_retry_cursor;
 
 	/* Info about the interface */
 	enum em_link_state	link_state;
@@ -674,6 +677,8 @@ void	igbv_initialize_transmit_unit(if_ctx_t);
 void	igbv_reconcile_mac(struct e1000_softc *, if_t);
 bool	igbv_reset(if_ctx_t);
 void	igbv_update_uc_addr_list(struct e1000_softc *, if_t);
+void	igbv_vlan_retry_add(struct e1000_softc *, u16);
+void	igbv_vlan_retry_clear(struct e1000_softc *, u16);
 
 /********************************************************************************
  * vendor_info_array

sys/dev/e1000/if_igbv.c

@@ -31,6 +31,8 @@
 #include <sys/sbuf.h>
 
 #define	IGBV_MAX_MAC_FILTERS	3
+#define	IGBV_VLAN_RETRY_BATCH	4
+#define	IGBV_VLAN_RETRY_WINDOW	(8 * SBT_1S)
 
 struct igb_vf_uc_addr_list {
 	struct e1000_softc	*sc;
@@ -38,6 +40,87 @@ struct igb_vf_uc_addr_list {
 };
 
 static bool	igbv_tx_pending(struct e1000_softc *);
+static bool	igbv_vlan_retry_pending(const struct e1000_softc *);
+static void	igbv_vlan_retry_tick(struct e1000_softc *);
+
+void
+igbv_vlan_retry_add(struct e1000_softc *sc, u16 vid)
+{
+	bool pending;
+
+	KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+	pending = igbv_vlan_retry_pending(sc);
+	sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f);
+	/* Bound the whole batch from its first failure, not each new VID. */
+	if (!pending)
+		sc->vf_vlan_retry_deadline =
+		    getsbinuptime() + IGBV_VLAN_RETRY_WINDOW;
+}
+
+void
+igbv_vlan_retry_clear(struct e1000_softc *sc, u16 vid)
+{
+
+	KASSERT(sc->vf_ifp, ("%s called for a PF", __func__));
+	sc->vf_vfta_retry[vid >> 5] &= ~(1U << (vid & 0x1f));
+}
+
+static bool
+igbv_vlan_retry_pending(const struct e1000_softc *sc)
+{
+	int i;
+
+	for (i = 0; i < EM_VFTA_SIZE; i++)
+		if (sc->vf_vfta_retry[i] != 0)
+			return (true);
+	return (false);
+}
+
+static void
+igbv_vlan_retry_tick(struct e1000_softc *sc)
+{
+	u32 bit;
+	u16 vid;
+	int attempts, i, remaining;
+
+	if (!igbv_vlan_retry_pending(sc)) {
+		sc->vf_vlan_retry_deadline = 0;
+		return;
+	}
+	if (getsbinuptime() >= sc->vf_vlan_retry_deadline) {
+		remaining = 0;
+		for (i = 0; i < EM_VFTA_SIZE; i++)
+			remaining += bitcount32(sc->vf_vfta_retry[i]);
+		memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry));
+		sc->vf_vlan_retry_deadline = 0;
+		device_printf(sc->dev,
+		    "VF VLAN restore retries exhausted for %d VIDs\n",
+		    remaining);
+		return;
+	}
+
+	/*
+	 * The mailbox NACK does not distinguish a transient PF rate limit
+	 * from permanent VLVF exhaustion.  Retry at the PF's sustained
+	 * allowance, but bound the entire recovery window so ENOSPC cannot
+	 * create a permanent mailbox poller.
+	 */
+	for (attempts = 0, i = 0;
+	    attempts < IGBV_VLAN_RETRY_BATCH && i < 4096; i++) {
+		vid = sc->vf_vlan_retry_cursor;
+		sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff;
+		bit = 1U << (vid & 0x1f);
+		if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0)
+			continue;
+		attempts++;
+		if ((sc->shadow_vfta[vid >> 5] & bit) == 0 ||
+		    e1000_vfta_set_vf(&sc->hw, vid, true) ==
+		    E1000_SUCCESS)
+			sc->vf_vfta_retry[vid >> 5] &= ~bit;
+	}
+	if (!igbv_vlan_retry_pending(sc))
+		sc->vf_vlan_retry_deadline = 0;
+}
 
 int
 igbv_if_attach_pre(if_ctx_t ctx)
@@ -98,7 +181,7 @@ igbv_if_update_admin_status(if_ctx_t ctx)
 	struct e1000_softc *sc;
 	struct e1000_hw *hw;
 	device_t dev;
-	bool link_check;
+	bool link_check, timer_tick;
 
 	sc = iflib_get_softc(ctx);
 	hw = &sc->hw;
@@ -151,9 +234,12 @@ igbv_if_update_admin_status(if_ctx_t ctx)
 		iflib_admin_intr_deferred(ctx);
 	}
 	/* em_if_init() establishes a new counter baseline after the reset. */
-	if (!sc->vf_reset_pending &&
-	    atomic_readandclear_32(&sc->stats_pending) != 0)
+	timer_tick = !sc->vf_reset_pending &&
+	    atomic_readandclear_32(&sc->stats_pending) != 0;
+	if (timer_tick) {
 		em_update_stats_counters(sc);
+		igbv_vlan_retry_tick(sc);
+	}
 }
 
 static bool
@@ -197,6 +283,9 @@ igbv_reset(if_ctx_t ctx)
 		return (false);
 	}
 	memset(sc->vf_vfta_stale, 0, sizeof(sc->vf_vfta_stale));
+	memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry));
+	sc->vf_vlan_retry_deadline = 0;
+	sc->vf_vlan_retry_cursor = 0;
 	if (e1000_init_hw(hw) < 0) {
 		device_printf(sc->dev, "Hardware Initialization Failed\n");
 		return (false);