imrcpan/freebsd
1
0 Fork 0
mirror of https://github.com/freebsd/freebsd-src.git synced 2026-10-28 11:25:21 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
a6999a1c947f3dc8b7491c903231efc2d78538d5
freebsd/sys/dev/e1000/if_igb_iov.c
T
Kevin Bowling a6999a1c94 net: Revert the ioctl SR-IOV VF status interface 
A native route Netlink interface will replace this stack.

Requested by:	glebius

This reverts commit 1ccf543b21.

This reverts commit 2c04cfa148.

This reverts commit 2d6114f6d2.

This reverts commit d15f2551b2.

This reverts commit ceb282bbd6.

This reverts commit c30021fe0d.

This reverts commit fb1820d23a.

This reverts commit 8696cc600f.
2026-08-10 15:49:55 -07:00

2204 lines
62 KiB
C
Raw Blame History

/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2010-2016, Intel Corporation
* Copyright (c) 2026 Kevin Bowling <[email protected]>
*/
#include "if_em.h"
#include "if_igb_iov.h"
#ifdef PCI_IOV
#include <sys/iov.h>
#include <sys/sdt.h>
#include <sys/time.h>
#define IGB_IOV_RAH_POOLSEL_SHIFT 18
#define IGB_IOV_RAH_POOLSEL_MASK (0xffU << IGB_IOV_RAH_POOLSEL_SHIFT)
#define IGB_IOV_MAX_MAC_FILTERS 3
#define IGB_IOV_MAX_MC_HASHES 30
#define IGB_IOV_MBX_RETRY_COUNT 6
/* Allow two complete 31-VID replays, then sustain eight additions/second. */
#define IGB_IOV_VLAN_CHANGE_BURST 64
#define IGB_IOV_VLAN_CHANGE_INTERVAL (SBT_1S / 8)
/* 82576 Datasheet rev. 2.0, Section 8.14.16: VMOLR[31] must be one. */
#define IGB_82576_VMOLR_RSV (1U << 31)
#define IGB_82576_LVMMC_BLOCK_MASK 0x1c
#define IGB_82576_NUM_QUEUES 16
#define IGB_82576_QUEUE_MASK 0xffff
#define IGB_82576_STAGGERED_QUEUE_SHIFT 8
#define IGB_82576_VF_QUEUE_STRIDE 8
#define IGB_82576_VF_QUEUES 2
#define IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE (1U << 2)
#define IGB_I350_LVMMC_MAC_VLAN_SPOOF (1U << 25)
#define IGB_I350_LVMMC_LAST_Q_SHIFT 29
#define IGB_I350_LVMMC_LAST_Q_MASK 0x7
#define IGB_I350_NUM_QUEUES 8
#define IGB_I350_QUEUE_MASK 0xff
#define IGB_I350_RESET_ACK_TIMEOUT (100 * SBT_1MS)
#define IGB_I350_VF_QUEUES 1
#define IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES 10
#define IGB_IOV_QUEUE_DISABLE_DELAY_US 10
#define IGB_IOV_QUEUE_DISABLE_PAUSE (100 * SBT_1US)
#define IGB_IOV_QUEUE_DISABLE_RETRIES 20
#define IGB_IOV_VF_QUEUES_MAX 2
#define IGB_VF_CTS (1U << 0)
#define IGB_VF_CAP_MAC (1U << 1)
#define IGB_VF_ACTIVE (1U << 2)
#define IGB_VF_MAC_ANTI_SPOOF (1U << 3)
#define IGB_VF_ALLOW_PROMISC (1U << 4)
#define IGB_VF_UCAST_PROMISC (1U << 5)
#define IGB_VF_MCAST_PROMISC (1U << 6)
#define IGB_VF_MCAST_OVERFLOW (1U << 7)
#define IGB_VF_MCAST_OVERFLOW_WARNED (1U << 8)
#define IGB_VF_MDD_BLOCKED (1U << 9)
#define IGB_VF_MBX_PENDING (1U << 10)
/*
* After bounded PFU retries, suppress future or overlapping VF requests until
* RST/VFLR starts a new mailbox epoch. Intel VF drivers assert CTRL.RST
* before sending their mailbox reset request.
*/
#define IGB_VF_MBX_GAVE_UP (1U << 11)
#define IGB_VF_MDD_NOTIFY_PENDING (1U << 12)
struct igb_vf {
u32 flags;
struct timeval last_nack;
struct timeval last_mbx_log;
struct timeval last_spoof_log;
struct timeval last_mdd_log;
struct timeval last_queue_log;
sbintime_t mbx_retry_at;
sbintime_t mdd_notify_at;
sbintime_t vlan_token_time;
u16 pool;
u16 rar_index;
u16 max_frame_size;
u16 mc_count;
u16 vlan_count;
u16 vlan_tokens;
u16 default_vlan;
u8 mbx_retry_count;
u8 mac[ETHER_ADDR_LEN];
u16 mc_hashes[IGB_IOV_MAX_MC_HASHES];
u32 vlans[EM_VFTA_SIZE];
};
struct igb_vf_mac_filter {
bool active;
u16 pool;
u16 rar_index;
u8 mac[ETHER_ADDR_LEN];
};
MALLOC_DEFINE(M_IGB_IOV, "igb_iov", "igb SR-IOV allocations");
/*
* These logical-write probes let hardware tests verify the elision policy.
* e1000_write_vfta_i350() expands one VFTA call into ten physical writes, so
* the probes intentionally count calls made by the rebuild rather than MMIO
* transactions. The state probe exposes the final software images while the
* stack arrays are still live.
*/
SDT_PROVIDER_DEFINE(igb_iov);
SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_clear,
"struct e1000_softc *", "u_int", "uint32_t");
SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vlvf_write,
"struct e1000_softc *", "u_int", "uint32_t");
SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, vfta_set,
"struct e1000_softc *", "u_int", "uint32_t");
SDT_PROBE_DEFINE3(igb_iov, vlan, rebuild, state,
"struct e1000_softc *", "uint32_t *", "uint32_t *");
SDT_PROBE_DEFINE4(igb_iov, mdd, sample, wvbr,
"struct e1000_softc *", "uint32_t", "uint32_t", "uint32_t");
static const struct timeval igb_iov_nack_interval = { 2, 0 };
static const struct timeval igb_iov_mbx_log_interval = { 2, 0 };
static const struct timeval igb_iov_spoof_log_interval = { 2, 0 };
static const struct timeval igb_iov_mdd_log_interval = { 2, 0 };
static const sbintime_t igb_iov_mdd_notify_retry = SBT_1S / 2;
static const sbintime_t igb_iov_mbx_retry_delay[IGB_IOV_MBX_RETRY_COUNT] = {
SBT_1MS,
2 * SBT_1MS,
4 * SBT_1MS,
8 * SBT_1MS,
16 * SBT_1MS,
32 * SBT_1MS,
};
static void igb_iov_clear_mac_filters(struct e1000_softc *,
const struct igb_vf *);
static bool igb_iov_mac_in_use(struct e1000_softc *, const u8 *,
const struct igb_vf *);
static bool igb_iov_vlan_present(struct e1000_softc *, u16, bool);
static int igb_iov_vlan_unique_count(struct e1000_softc *, bool);
static void
igb_iov_mbx_retry_callout(void *arg)
{
struct e1000_softc *sc;
sc = arg;
/*
* Mailbox service is serialized by iflib's context lock. The
* callout only re-enters through the ordinary admin task.
*/
iflib_admin_intr_deferred(sc->ctx);
}
static u_int
igb_iov_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx)
{
u8 *mta;
if (idx == MAX_NUM_MULTICAST_ADDRESSES)
return (0);
mta = arg;
memcpy(&mta[idx * ETHER_ADDR_LEN], LLADDR(sdl), ETHER_ADDR_LEN);
return (1);
}
static bool
igb_iov_pf_vlan_promisc(struct e1000_softc *sc)
{
if_t ifp;
ifp = iflib_get_ifp(sc->ctx);
return (sc->iov_pf_vlan_promisc ||
(if_getflags(ifp) & IFF_PROMISC) != 0);
}
static bool
igb_iov_mac_valid(const u8 *mac)
{
static const u8 zero[ETHER_ADDR_LEN];
return (!ETHER_IS_MULTICAST(mac) &&
memcmp(mac, zero, ETHER_ADDR_LEN) != 0);
}
static bool
igb_iov_nack_allowed(struct igb_vf *vf)
{
return (ratecheck(&vf->last_nack, &igb_iov_nack_interval) != 0);
}
static void
igb_iov_reset_vlan_rate(struct igb_vf *vf)
{
vf->vlan_token_time = getsbinuptime();
vf->vlan_tokens = IGB_IOV_VLAN_CHANGE_BURST;
}
static bool
igb_iov_vlan_add_allowed(struct igb_vf *vf)
{
sbintime_t elapsed, now;
uint64_t refill;
now = getsbinuptime();
elapsed = now - vf->vlan_token_time;
if (elapsed >= IGB_IOV_VLAN_CHANGE_INTERVAL) {
refill = elapsed / IGB_IOV_VLAN_CHANGE_INTERVAL;
vf->vlan_tokens = min((uint64_t)IGB_IOV_VLAN_CHANGE_BURST,
vf->vlan_tokens + refill);
vf->vlan_token_time = now;
}
if (vf->vlan_tokens == 0)
return (false);
vf->vlan_tokens--;
return (true);
}
static u32
igb_iov_reply_header(u32 request, bool cts, bool ack)
{
u32 reply, type;
type = request & 0xffff;
if (type == E1000_VF_SET_MAC_ADDR &&
(request & E1000_VT_MSGINFO_MASK) != 0)
reply = request;
else
reply = type;
reply &= ~(E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK |
E1000_VT_MSGTYPE_CTS);
if (cts)
reply |= E1000_VT_MSGTYPE_CTS;
reply |= ack ? E1000_VT_MSGTYPE_ACK : E1000_VT_MSGTYPE_NACK;
return (reply);
}
bool
igb_iov_supported(const struct e1000_softc *sc)
{
switch (sc->hw.mac.type) {
case e1000_82576:
case e1000_i350:
return (true);
default:
return (false);
}
}
bool
igb_iov_enabled(const struct e1000_softc *sc)
{
return (sc->num_vfs != 0);
}
int
igb_iov_attach(struct e1000_softc *sc)
{
nvlist_t *pf_schema, *vf_schema;
int error, iov_pos;
if (!igb_iov_supported(sc))
return (0);
if (pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos) != 0)
return (0);
pf_schema = pci_iov_schema_alloc_node();
vf_schema = pci_iov_schema_alloc_node();
pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
IOV_SCHEMA_HASDEFAULT, true);
pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
IOV_SCHEMA_HASDEFAULT, false);
pci_iov_schema_add_bool(vf_schema, "allow-promisc",
IOV_SCHEMA_HASDEFAULT, false);
pci_iov_schema_add_vlan(vf_schema, "vlan", IOV_SCHEMA_HASDEFAULT,
VF_VLAN_TRUNK);
error = pci_iov_attach(sc->dev, pf_schema, vf_schema);
if (error != 0)
device_printf(sc->dev,
"failed to attach SR-IOV configuration interface: %d\n",
error);
else {
callout_init(&sc->iov_mbx_retry, 1);
sc->iov_mbx_retry_initialized = true;
}
return (error);
}
void
igb_iov_detach(struct e1000_softc *sc)
{
if (!sc->iov_mbx_retry_initialized)
return;
callout_drain(&sc->iov_mbx_retry);
sc->iov_mbx_retry_initialized = false;
}
static u32
igb_iov_active_mask(struct e1000_softc *sc)
{
u32 mask;
int i;
mask = 0;
for (i = 0; i < sc->num_vfs; i++)
if (sc->vfs[i].flags & IGB_VF_ACTIVE)
mask |= 1U << i;
return (mask);
}
static void
igb_iov_map_rar(struct e1000_softc *sc, u16 rar, const u8 *mac, u16 pool)
{
struct e1000_hw *hw;
u32 rah;
hw = &sc->hw;
e1000_rar_set(hw, __DECONST(u8 *, mac), rar);
rah = E1000_READ_REG(hw, E1000_RAH(rar));
rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
rah |= 1U << (IGB_IOV_RAH_POOLSEL_SHIFT + pool);
E1000_WRITE_REG(hw, E1000_RAH(rar), rah);
}
static void
igb_iov_clear_rar(struct e1000_softc *sc, u16 rar)
{
u8 zero[ETHER_ADDR_LEN] = {};
e1000_rar_set(&sc->hw, zero, rar);
}
static void
igb_iov_clear_mac_filters(struct e1000_softc *sc, const struct igb_vf *vf)
{
struct igb_vf_mac_filter *filter;
int i;
for (i = 0; i < sc->num_vf_mac_filters; i++) {
filter = &sc->vf_mac_filters[i];
if (!filter->active || filter->pool != vf->pool)
continue;
igb_iov_clear_rar(sc, filter->rar_index);
filter->active = false;
memset(filter->mac, 0, sizeof(filter->mac));
}
}
static u32
igb_iov_switch_reg(struct e1000_softc *sc)
{
return (sc->hw.mac.type == e1000_82576 ?
E1000_DTXSWC : E1000_TXSWC);
}
static void
igb_iov_set_anti_spoof(struct e1000_softc *sc, struct igb_vf *vf)
{
struct e1000_hw *hw;
u32 reg, value;
hw = &sc->hw;
reg = igb_iov_switch_reg(sc);
value = E1000_READ_REG(hw, reg);
value &= ~((1U << vf->pool) |
(1U << (vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
if (vf->flags & IGB_VF_MAC_ANTI_SPOOF)
value |= 1U << vf->pool;
if (vf->flags & IGB_VF_ACTIVE)
value |= 1U <<
(vf->pool + E1000_DTXSWC_VLAN_SPOOF_SHIFT);
E1000_WRITE_REG(hw, reg, value);
}
static void
igb_iov_set_uta(struct e1000_softc *sc)
{
struct e1000_hw *hw;
bool enable;
int i;
if (!igb_iov_enabled(sc) || sc->hw.mac.type != e1000_82576)
return;
hw = &sc->hw;
enable = (E1000_READ_REG(hw, E1000_VMOLR(sc->pool)) &
E1000_VMOLR_ROPE) != 0;
for (i = 0; i < sc->num_vfs; i++)
if ((sc->vfs[i].flags &
(IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) ==
(IGB_VF_ACTIVE | IGB_VF_UCAST_PROMISC)) {
enable = true;
break;
}
for (i = 0; i < MAX_MTA_REG; i++)
E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i,
enable ? 0xffffffffU : 0);
}
static void
igb_iov_configure_dvmolr(struct e1000_softc *sc, u16 pool,
bool strip_vlan, bool hide_vlan, bool vf_pool)
{
struct e1000_hw *hw;
u32 dvmolr;
hw = &sc->hw;
if (hw->mac.type != e1000_i350)
return;
dvmolr = E1000_READ_REG(hw, E1000_DVMOLR(pool));
dvmolr &= ~(E1000_DVMOLR_HIDVLAN | E1000_DVMOLR_STRVLAN |
E1000_DVMOLR_STRCRC);
if (hide_vlan)
dvmolr |= E1000_DVMOLR_HIDVLAN;
if (strip_vlan)
dvmolr |= E1000_DVMOLR_STRVLAN;
if (vf_pool || strip_vlan ||
(E1000_READ_REG(hw, E1000_RCTL) & E1000_RCTL_SECRC) != 0)
dvmolr |= E1000_DVMOLR_STRCRC;
E1000_WRITE_REG(hw, E1000_DVMOLR(pool), dvmolr);
}
static void
igb_iov_configure_vmolr(struct e1000_softc *sc, struct igb_vf *vf)
{
struct e1000_hw *hw;
u32 max_frame_size, vmolr, vmvir;
hw = &sc->hw;
max_frame_size = vf->max_frame_size;
if (vf->vlan_count != 0)
max_frame_size = min(max_frame_size + VLAN_TAG_SIZE,
IGB_IOV_MAX_FRAME_SIZE);
vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf->pool));
vmolr &= ~(E1000_VMOLR_RLPML_MASK | E1000_VMOLR_RSSE |
E1000_VMOLR_VPE | E1000_VMOLR_UPE | E1000_VMOLR_ROMPE |
E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_STRVLAN);
vmolr |= E1000_VMOLR_BAM | E1000_VMOLR_LPE |
(max_frame_size & E1000_VMOLR_RLPML_MASK);
if (vf->default_vlan == 0)
vmolr |= E1000_VMOLR_AUPE;
if (vf->mc_count != 0 &&
(vf->flags & (IGB_VF_MCAST_PROMISC |
IGB_VF_MCAST_OVERFLOW)) == 0)
vmolr |= E1000_VMOLR_ROMPE;
if (hw->mac.type == e1000_82576)
vmolr |= IGB_82576_VMOLR_RSV;
if (vf->flags & IGB_VF_UCAST_PROMISC) {
if (hw->mac.type == e1000_82576)
vmolr |= E1000_VMOLR_ROPE;
else
vmolr |= E1000_VMOLR_UPE;
}
/*
* The mailbox can describe only 30 hashes. Fall back to receiving all
* multicast within the VF's VLAN membership when that list overflows.
*/
if ((vf->flags & (IGB_VF_MCAST_PROMISC |
IGB_VF_MCAST_OVERFLOW)) != 0)
vmolr |= E1000_VMOLR_MPME;
if (hw->mac.type == e1000_82576 && vf->vlan_count != 0)
vmolr |= E1000_VMOLR_STRVLAN;
/* A nonzero default VLAN makes this VF an untagged access port. */
if (vf->default_vlan == 0)
vmvir = 0;
else
vmvir = vf->default_vlan | E1000_VMVIR_VLANA_DEFAULT;
E1000_WRITE_REG(hw, E1000_VMOLR(vf->pool), vmolr);
E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), vmvir);
igb_iov_configure_dvmolr(sc, vf->pool, vf->vlan_count != 0,
vf->default_vlan != 0, true);
}
static void
igb_iov_configure_pf_vmolr(struct e1000_softc *sc)
{
struct e1000_hw *hw;
if_t ifp;
bool strip_vlan;
u32 max_frame_size;
u32 old_vmolr, vmolr;
hw = &sc->hw;
ifp = iflib_get_ifp(sc->ctx);
max_frame_size = min(sc->shared->isc_max_frame_size + VLAN_TAG_SIZE,
IGB_IOV_MAX_FRAME_SIZE);
strip_vlan = (E1000_READ_REG(hw, E1000_CTRL) & E1000_CTRL_VME) != 0;
old_vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
vmolr = E1000_VMOLR_BAM | E1000_VMOLR_AUPE |
E1000_VMOLR_LPE |
(max_frame_size & E1000_VMOLR_RLPML_MASK);
if (hw->mac.type == e1000_82576) {
vmolr |= IGB_82576_VMOLR_RSV;
if (strip_vlan)
vmolr |= E1000_VMOLR_STRVLAN;
} else
vmolr |= old_vmolr & E1000_VMOLR_VPE;
if (if_getflags(ifp) & IFF_PROMISC) {
if (hw->mac.type == e1000_82576)
vmolr |= E1000_VMOLR_ROPE;
else
vmolr |= E1000_VMOLR_UPE | E1000_VMOLR_VPE;
vmolr |= E1000_VMOLR_MPME;
} else if ((if_getflags(ifp) & IFF_ALLMULTI) != 0 ||
if_llmaddr_count(ifp) >= MAX_NUM_MULTICAST_ADDRESSES)
vmolr |= E1000_VMOLR_MPME;
else if (if_llmaddr_count(ifp) != 0)
vmolr |= E1000_VMOLR_ROMPE;
E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);
igb_iov_configure_dvmolr(sc, sc->pool, strip_vlan, false, false);
}
void
igb_iov_update_pf_vmolr(struct e1000_softc *sc)
{
if (!igb_iov_enabled(sc))
return;
igb_iov_configure_pf_vmolr(sc);
igb_iov_set_uta(sc);
}
u32
igb_iov_intr_mask(const struct e1000_softc *sc)
{
if (!sc->iov_hw_active)
return (0);
return (E1000_IMS_VMMB | E1000_IMS_MDDET);
}
void
igb_iov_intr_drain_stale(struct e1000_softc *sc)
{
struct e1000_hw *hw;
u32 icr;
if (atomic_readandclear_32(&sc->iov_intr_drain_pending) == 0)
return;
hw = &sc->hw;
/*
* Consume setup-time diagnostic state at the actual transition from
* masked to armed. Read ICR last so an event arriving after the drain
* remains pending and is delivered when the caller enables MDDET.
*/
(void)E1000_READ_REG(hw, E1000_LVMMC);
if (hw->mac.type == e1000_82576)
(void)E1000_READ_REG(hw, E1000_WVBR);
icr = E1000_READ_REG(hw, E1000_ICR);
/*
* em_if_init() injects LSC after IOV setup to close the post-reset
* link race. Preserve that cause across this MDDET-specific drain.
*/
if (__predict_true(icr != 0xffffffff) &&
(icr & E1000_ICR_LSC) != 0)
E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_LSC);
}
static void
igb_iov_vfta_shadow_invalidate(struct e1000_softc *sc)
{
/*
* I350 erratum 20 makes VFTA reads unreliable while VMDq loopback or
* anti-spoofing is active. The shadow is therefore authoritative
* until a reset or another independent hardware writer invalidates
* it. Readback cannot reliably audit a stale-but-valid shadow on
* this part, so keep all shadow mutation in these two helpers.
*/
memset(sc->iov_vfta, 0, sizeof(sc->iov_vfta));
sc->iov_vfta_valid = false;
}
static void
igb_iov_vfta_shadow_store(struct e1000_softc *sc, const u32 *vfta)
{
memcpy(sc->iov_vfta, vfta, sizeof(sc->iov_vfta));
sc->iov_vfta_valid = true;
}
static void
igb_iov_notify_vfs_reset(struct e1000_softc *sc)
{
struct igb_vf *vf;
struct e1000_hw *hw;
sbintime_t deadline;
u32 msg, pending, undelivered;
int i;
hw = &sc->hw;
/*
* Process VFLRs first and wait only for VFs that completed their
* mailbox handshake. An unattached VF has nobody who can acknowledge.
*/
igb_iov_handle_mbx(sc);
pending = 0;
for (i = 0; i < sc->num_vfs; i++) {
vf = &sc->vfs[i];
if ((vf->flags & (IGB_VF_ACTIVE | IGB_VF_CTS)) ==
(IGB_VF_ACTIVE | IGB_VF_CTS))
pending |= 1U << i;
}
if (pending == 0)
return;
/*
* I350 SDM section 4.6.11.2.3 requires each VF to acknowledge a
* mailbox warning before the PF asserts CTRL.RST.
*
* The mailbox pass above drained requests and stale acknowledgements.
* A VF read of the new notification sets its ACK bit.
*/
undelivered = 0;
for (i = 0; i < sc->num_vfs; i++) {
if ((pending & (1U << i)) == 0)
continue;
msg = E1000_PF_CONTROL_MSG;
if (e1000_write_mbx(hw, &msg, 1, i) != 0) {
undelivered |= 1U << i;
pending &= ~(1U << i);
}
}
if (undelivered != 0)
device_printf(sc->dev,
"could not deliver reset warning to VF mask %#x\n",
undelivered);
deadline = getsbinuptime() + IGB_I350_RESET_ACK_TIMEOUT;
while (pending != 0 && getsbinuptime() < deadline) {
for (i = 0; i < sc->num_vfs; i++) {
if ((pending & (1U << i)) != 0 &&
e1000_check_for_ack(hw, i) == 0)
pending &= ~(1U << i);
}
if (pending != 0)
pause_sbt("igback", SBT_1MS, 0, C_HARDCLOCK);
}
if (pending != 0)
device_printf(sc->dev,
"VF reset acknowledgement timed out for mask %#x\n",
pending);
}
void
igb_iov_reset_prepare(struct e1000_softc *sc)
{
struct e1000_hw *hw;
u32 mask;
if (sc->iov_hw_active) {
hw = &sc->hw;
if (atomic_load_acq_32(&sc->iov_teardown) == 0) {
if (hw->mac.type == e1000_i350)
igb_iov_notify_vfs_reset(sc);
else
igb_iov_ping_all_vfs(sc);
}
/* Stop VF DMA before the PF asserts CTRL.RST. */
mask = 1U << sc->pool;
E1000_WRITE_REG(hw, E1000_VFRE, mask);
E1000_WRITE_REG(hw, E1000_VFTE, mask);
E1000_WRITE_FLUSH(hw);
}
sc->iov_hw_active = false;
if (sc->iov_mbx_retry_initialized)
callout_stop(&sc->iov_mbx_retry);
sc->iov_mta_valid = false;
igb_iov_vfta_shadow_invalidate(sc);
atomic_readandclear_32(&sc->iov_mdd_cause);
atomic_readandclear_32(&sc->iov_pending);
atomic_readandclear_32(&sc->iov_spoof_pending);
atomic_readandclear_32(&sc->iov_blocked_pending);
/*
* Normal iflib initialization prepares the reset before
* igb_iov_initialize() requests this drain. Preserve a still-pending
* I350 request across a later stop or repeated preparation so the next
* interrupt arm consumes it. Other families retain the ordinary
* stop-time cleanup.
*/
if (sc->hw.mac.type != e1000_i350)
atomic_readandclear_32(&sc->iov_intr_drain_pending);
}
void
igb_iov_rebuild_mta(struct e1000_softc *sc)
{
struct e1000_hw *hw;
struct igb_vf *vf;
u32 hash_bit, hash_reg, hash_value;
u32 mta[MAX_MTA_REG] = {};
u16 hash;
bool changed;
int i, j, mcnt;
if (!igb_iov_enabled(sc))
return;
hw = &sc->hw;
memset(sc->mta, 0,
ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
mcnt = if_foreach_llmaddr(iflib_get_ifp(sc->ctx),
igb_iov_copy_maddr, sc->mta);
mcnt = min(mcnt, MAX_NUM_MULTICAST_ADDRESSES);
for (i = 0; i < mcnt; i++) {
hash_value = e1000_hash_mc_addr(hw,
&sc->mta[i * ETHER_ADDR_LEN]);
hash_reg = (hash_value >> 5) &
(hw->mac.mta_reg_count - 1);
hash_bit = hash_value & 0x1f;
mta[hash_reg] |= 1U << hash_bit;
}
for (i = 0; i < sc->num_vfs; i++) {
vf = &sc->vfs[i];
if (!(vf->flags & IGB_VF_ACTIVE))
continue;
for (j = 0; j < vf->mc_count; j++) {
hash = vf->mc_hashes[j] & 0xfff;
mta[(hash >> 5) & (hw->mac.mta_reg_count - 1)] |=
1U << (hash & 0x1f);
}
}
changed = false;
for (i = hw->mac.mta_reg_count - 1; i >= 0; i--) {
if (sc->iov_mta_valid && hw->mac.mta_shadow[i] == mta[i])
continue;
hw->mac.mta_shadow[i] = mta[i];
E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, mta[i]);
changed = true;
}
if (changed)
E1000_WRITE_FLUSH(hw);
sc->iov_mta_valid = true;
}
static int
igb_iov_vlvf_add(u32 *vlvf, const u32 *old_vlvf, u16 vid, u16 pool,
bool preserve_only)
{
int free_slot, i;
free_slot = -1;
for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
(vlvf[i] & E1000_VLVF_VLANID_MASK) == vid) {
vlvf[i] |= 1U << (E1000_VLVF_POOLSEL_SHIFT + pool);
return (0);
}
if (free_slot == -1 &&
(vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0)
free_slot = i;
}
for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
if ((old_vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0 &&
(old_vlvf[i] & E1000_VLVF_VLANID_MASK) == vid &&
(vlvf[i] & E1000_VLVF_VLANID_ENABLE) == 0) {
free_slot = i;
break;
}
if (preserve_only && (i == E1000_VLVF_ARRAY_SIZE))
return (ENOENT);
if (free_slot == -1)
return (ENOSPC);
vlvf[free_slot] = E1000_VLVF_VLANID_ENABLE | vid |
(1U << (E1000_VLVF_POOLSEL_SHIFT + pool));
return (0);
}
void
igb_iov_rebuild_vlan(struct e1000_softc *sc)
{
struct e1000_hw *hw;
struct igb_vf *vf;
u32 old_vlvf[E1000_VLVF_ARRAY_SIZE];
u32 effective_vfta[EM_VFTA_SIZE], vfta[EM_VFTA_SIZE];
u32 vlvf[E1000_VLVF_ARRAY_SIZE];
u32 old_vfta, rctl, vmolr;
bool force_vfta, pf_overflow, pf_vlan_promisc, preserve_pf;
bool vfta_changed, vlvf_changed;
int i, vid;
if (!igb_iov_enabled(sc))
return;
hw = &sc->hw;
rctl = E1000_READ_REG(hw, E1000_RCTL);
rctl &= ~E1000_RCTL_CFIEN;
rctl |= E1000_RCTL_VFE;
E1000_WRITE_REG(hw, E1000_RCTL, rctl);
memcpy(vfta, sc->shadow_vfta, sizeof(vfta));
memset(vlvf, 0, sizeof(vlvf));
for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
old_vlvf[i] = E1000_READ_REG(hw, E1000_VLVF(i));
pf_vlan_promisc = igb_iov_pf_vlan_promisc(sc);
pf_overflow = !pf_vlan_promisc && hw->mac.type == e1000_i350 &&
igb_iov_vlan_unique_count(sc, true) > E1000_VLVF_ARRAY_SIZE;
preserve_pf = !pf_vlan_promisc && !pf_overflow;
/* First keep every surviving VF mapping in its current slot. */
for (i = 0; i < sc->num_vfs; i++) {
vf = &sc->vfs[i];
if (!(vf->flags & IGB_VF_ACTIVE))
continue;
for (vid = 0; vid < 4096; vid++) {
if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
0)
continue;
(void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
vf->pool, true);
}
}
/*
* Preserve PF mappings unless I350 needs their slots for VFs.
* PF-only VLANs on 82576 intentionally have no VLVF mapping and
* reach the default PF pool after passing the global VFTA.
*/
if (preserve_pf)
for (vid = 0; vid < 4096; vid++) {
if ((sc->shadow_vfta[vid >> 5] &
(1U << (vid & 0x1f))) == 0)
continue;
if (hw->mac.type == e1000_82576 &&
!igb_iov_vlan_present(sc, vid, false))
continue;
(void)igb_iov_vlvf_add(vlvf, old_vlvf, vid,
sc->pool, true);
}
/* Allocate new VF mappings before PF mappings. */
for (i = 0; i < sc->num_vfs; i++) {
vf = &sc->vfs[i];
if (!(vf->flags & IGB_VF_ACTIVE))
continue;
for (vid = 0; vid < 4096; vid++) {
if ((vf->vlans[vid >> 5] & (1U << (vid & 0x1f))) ==
0)
continue;
if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
vf->pool, false) == 0)
vfta[vid >> 5] |= 1U << (vid & 0x1f);
}
igb_iov_configure_vmolr(sc, vf);
}
if (!pf_vlan_promisc)
for (vid = 0; vid < 4096; vid++) {
if ((sc->shadow_vfta[vid >> 5] &
(1U << (vid & 0x1f))) == 0)
continue;
/*
* With no VLVF match, 82576 sends a globally admitted
* VLAN to the default PF pool. A VLVF entry is needed
* only when this VLAN is also assigned to a VF.
*/
if (hw->mac.type == e1000_82576 &&
!igb_iov_vlan_present(sc, vid, false))
continue;
if (igb_iov_vlvf_add(vlvf, old_vlvf, vid,
sc->pool, false) != 0)
pf_overflow = true;
}
if (pf_vlan_promisc) {
memset(vfta, 0xff, sizeof(vfta));
for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
if ((vlvf[i] & E1000_VLVF_VLANID_ENABLE) != 0)
vlvf[i] |= 1U <<
(E1000_VLVF_POOLSEL_SHIFT + sc->pool);
}
/*
* Establish the PF fallback before an overflowing I350 rebuild can
* displace one of its old VLVF mappings.
*/
vmolr = E1000_READ_REG(hw, E1000_VMOLR(sc->pool));
vmolr &= ~E1000_VMOLR_VPE;
if (hw->mac.type == e1000_i350 &&
(pf_overflow || pf_vlan_promisc))
vmolr |= E1000_VMOLR_VPE;
E1000_WRITE_REG(hw, E1000_VMOLR(sc->pool), vmolr);
/*
* Remove global VFTA membership before removing a VLAN entirely, and
* add a VLVF mapping before globally admitting a new VF VLAN. A
* transition to a PF-only VLAN deliberately retains VFTA membership
* and falls through to the default PF pool.
*/
force_vfta = hw->mac.type == e1000_i350 &&
!sc->iov_vfta_valid;
vfta_changed = false;
for (i = 0; i < EM_VFTA_SIZE; i++) {
/*
* I350 erratum 20 makes VFTA reads unreliable while VMDq
* loopback or anti-spoofing is active. Its ten-write
* workaround is already in e1000_write_vfta_i350(). Force a
* complete clear when the authoritative shadow is invalid;
* 82576 can safely diff against its live register contents.
*/
if (hw->mac.type == e1000_i350)
old_vfta = force_vfta ? 0 : sc->iov_vfta[i];
else
old_vfta =
E1000_READ_REG_ARRAY(hw, E1000_VFTA, i);
effective_vfta[i] = old_vfta & vfta[i];
if (force_vfta || effective_vfta[i] != old_vfta) {
SDT_PROBE3(igb_iov, vlan, rebuild, vfta_clear,
sc, i, effective_vfta[i]);
e1000_write_vfta(hw, i, effective_vfta[i]);
vfta_changed = true;
}
}
if (vfta_changed)
E1000_WRITE_FLUSH(hw);
vlvf_changed = false;
for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
if (vlvf[i] != old_vlvf[i]) {
SDT_PROBE3(igb_iov, vlan, rebuild, vlvf_write,
sc, i, vlvf[i]);
E1000_WRITE_REG(hw, E1000_VLVF(i), vlvf[i]);
vlvf_changed = true;
}
if (vlvf_changed)
E1000_WRITE_FLUSH(hw);
vfta_changed = false;
for (i = 0; i < EM_VFTA_SIZE; i++)
if (vfta[i] != effective_vfta[i]) {
SDT_PROBE3(igb_iov, vlan, rebuild, vfta_set,
sc, i, vfta[i]);
e1000_write_vfta(hw, i, vfta[i]);
vfta_changed = true;
}
if (vfta_changed)
E1000_WRITE_FLUSH(hw);
SDT_PROBE3(igb_iov, vlan, rebuild, state, sc, vfta, vlvf);
igb_iov_vfta_shadow_store(sc, vfta);
}
static bool
igb_iov_vlan_present(struct e1000_softc *sc, u16 vid, bool include_pf)
{
int i;
if (include_pf &&
(sc->shadow_vfta[vid >> 5] & (1U << (vid & 0x1f))) != 0)
return (true);
for (i = 0; i < sc->num_vfs; i++)
if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
(sc->vfs[i].vlans[vid >> 5] &
(1U << (vid & 0x1f))) != 0)
return (true);
return (false);
}
static int
igb_iov_vlan_unique_count(struct e1000_softc *sc, bool include_pf)
{
u32 vlans;
int count, i, word;
count = 0;
for (word = 0; word < EM_VFTA_SIZE; word++) {
vlans = include_pf ? sc->shadow_vfta[word] : 0;
for (i = 0; i < sc->num_vfs; i++)
if ((sc->vfs[i].flags & IGB_VF_ACTIVE) != 0)
vlans |= sc->vfs[i].vlans[word];
count += bitcount32(vlans);
}
return (count);
}
static int
igb_iov_set_vlan(struct e1000_softc *sc, struct igb_vf *vf, u16 vid,
bool add)
{
u32 bit;
bool present;
bit = 1U << (vid & 0x1f);
present = (vf->vlans[vid >> 5] & bit) != 0;
if (vid == 0) {
if (!present) {
vf->vlans[0] |= 1U;
igb_iov_rebuild_vlan(sc);
}
return (0);
}
if (add == present)
return (0);
/*
* Removals always reduce privilege and remain available. Charge only
* additions, which a hostile VF must alternate with removals to force
* repeated global VLAN rebuilds.
*/
if (add && !igb_iov_vlan_present(sc, vid, false) &&
igb_iov_vlan_unique_count(sc, false) >=
E1000_VLVF_ARRAY_SIZE)
return (ENOSPC);
if (add && !igb_iov_vlan_add_allowed(vf))
return (EBUSY);
if (add) {
vf->vlans[vid >> 5] |= bit;
vf->vlan_count++;
} else {
vf->vlans[vid >> 5] &= ~bit;
vf->vlan_count--;
}
igb_iov_rebuild_vlan(sc);
return (0);
}
static void
igb_iov_reset_vf_state(struct e1000_softc *sc, struct igb_vf *vf)
{
bool update_uta;
update_uta = (vf->flags & IGB_VF_UCAST_PROMISC) != 0;
vf->flags &= ~(IGB_VF_CTS | IGB_VF_UCAST_PROMISC |
IGB_VF_MCAST_PROMISC | IGB_VF_MCAST_OVERFLOW |
IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP |
IGB_VF_MDD_NOTIFY_PENDING);
vf->mbx_retry_at = 0;
vf->mdd_notify_at = 0;
vf->mbx_retry_count = 0;
/*
* A reset starts a new mailbox epoch. Permit one immediate NACK so a
* premature non-reset request does not wait for its posted-read
* timeout.
*/
memset(&vf->last_nack, 0, sizeof(vf->last_nack));
vf->max_frame_size = ETHER_MAX_LEN;
vf->mc_count = 0;
vf->vlan_count = 0;
memset(vf->mc_hashes, 0, sizeof(vf->mc_hashes));
memset(vf->vlans, 0, sizeof(vf->vlans));
/* Preserve the administrative access VLAN across VF and PF resets. */
if (vf->default_vlan == 0)
vf->vlans[0] = 1U;
else {
vf->vlans[vf->default_vlan >> 5] =
1U << (vf->default_vlan & 0x1f);
vf->vlan_count = 1;
}
igb_iov_configure_vmolr(sc, vf);
if (update_uta)
igb_iov_set_uta(sc);
}
static bool
igb_iov_vf_vlan_is_default(const struct igb_vf *vf)
{
u32 expected;
int i;
for (i = 0; i < EM_VFTA_SIZE; i++) {
expected = 0;
if (i == vf->default_vlan >> 5)
expected = 1U << (vf->default_vlan & 0x1f);
if (vf->vlans[i] != expected)
return (false);
}
return (true);
}
static bool
igb_iov_sanitize_vf_queues(struct e1000_softc *sc,
struct igb_vf *vf)
{
struct e1000_hw *hw;
u16 qid[IGB_IOV_VF_QUEUES_MAX];
u32 rxdctl, txdctl;
int i, nqueues, retry;
hw = &sc->hw;
switch (hw->mac.type) {
case e1000_82576:
nqueues = IGB_82576_VF_QUEUES;
qid[0] = vf->pool;
qid[1] = vf->pool + IGB_82576_VF_QUEUE_STRIDE;
break;
case e1000_i350:
nqueues = IGB_I350_VF_QUEUES;
qid[0] = vf->pool;
break;
default:
return (true);
}
/*
* I350 maps pool n to queue n. 82576 gives VF n physical queues n
* and n + 8, so both retained queue configurations must be cleared.
*/
for (i = 0; i < nqueues; i++)
KASSERT(qid[i] < (hw->mac.type == e1000_82576 ?
IGB_82576_NUM_QUEUES : IGB_I350_NUM_QUEUES),
("%s: invalid VF queue %u", __func__, qid[i]));
/*
* The 82576 and I350 specification updates, Software Clarification 3,
* note that VFLR does not reset the VF queue configuration. Clear the
* PF-programmable state before acknowledging the reset so a new VF
* owner cannot inherit it, particularly a descriptor-head write-back
* DMA address. The new VF driver initializes its active ring pointers
* during queue setup.
*
* Disable every queue first, then wait for outstanding DMA activity to
* stop before clearing TDWBAL/H and the remaining retained state.
* Spin only for the normal fast transition, then sleep so a VF that
* keeps asserting QUEUE_ENABLE cannot busy-wait the PF for 10 ms.
*/
for (i = 0; i < nqueues; i++) {
E1000_WRITE_REG(hw, E1000_RXDCTL(qid[i]), 0);
E1000_WRITE_REG(hw, E1000_TXDCTL(qid[i]), 0);
}
E1000_WRITE_FLUSH(hw);
for (retry = 0; retry < IGB_IOV_QUEUE_DISABLE_RETRIES; retry++) {
for (i = 0; i < nqueues; i++) {
rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(qid[i]));
txdctl = E1000_READ_REG(hw, E1000_TXDCTL(qid[i]));
if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 ||
(txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0)
break;
}
if (i == nqueues)
break;
if (retry + 1 < IGB_IOV_QUEUE_DISABLE_RETRIES) {
if (retry < IGB_IOV_QUEUE_DISABLE_BUSY_RETRIES)
DELAY(IGB_IOV_QUEUE_DISABLE_DELAY_US);
else
pause_sbt("igbqds",
IGB_IOV_QUEUE_DISABLE_PAUSE, 0,
C_PREL(1));
}
}
if (retry == IGB_IOV_QUEUE_DISABLE_RETRIES) {
if (ratecheck(&vf->last_queue_log,
&igb_iov_mbx_log_interval))
device_printf(sc->dev,
"could not disable queues for VF %u; "
"reset deferred\n", vf->pool);
return (false);
}
for (i = 0; i < nqueues; i++) {
E1000_WRITE_REG(hw, E1000_SRRCTL(qid[i]), 0);
E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(qid[i]), 0);
E1000_WRITE_REG(hw, E1000_TDWBAL(qid[i]), 0);
E1000_WRITE_REG(hw, E1000_TDWBAH(qid[i]), 0);
E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(qid[i]), 0);
}
E1000_WRITE_REG(hw, E1000_PSRTYPE(vf->pool), 0);
E1000_WRITE_FLUSH(hw);
return (true);
}
static bool
igb_iov_reset_event_common(struct e1000_softc *sc, struct igb_vf *vf,
bool reset_intrs)
{
struct e1000_hw *hw;
bool rebuild_mta, rebuild_vlan, sanitized;
u32 reg;
hw = &sc->hw;
rebuild_mta = vf->mc_count != 0;
rebuild_vlan = !igb_iov_vf_vlan_is_default(vf);
reg = E1000_READ_REG(hw, E1000_VFTE);
E1000_WRITE_REG(hw, E1000_VFTE, reg & ~(1U << vf->pool));
reg = E1000_READ_REG(hw, E1000_VFRE);
E1000_WRITE_REG(hw, E1000_VFRE, reg & ~(1U << vf->pool));
if (reset_intrs)
E1000_WRITE_REG(hw, E1000_VTCTRL(vf->pool),
E1000_VTCTRL_RST);
sanitized = igb_iov_sanitize_vf_queues(sc, vf);
E1000_WRITE_REG(hw, E1000_VMVIR(vf->pool), 0);
igb_iov_clear_mac_filters(sc, vf);
igb_iov_clear_rar(sc, vf->rar_index);
igb_iov_reset_vf_state(sc, vf);
if (rebuild_mta)
igb_iov_rebuild_mta(sc);
if (rebuild_vlan)
igb_iov_rebuild_vlan(sc);
return (sanitized);
}
static bool
igb_iov_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
{
return (igb_iov_reset_event_common(sc, vf, true));
}
static void
igb_iov_mdd_reset_event(struct e1000_softc *sc, struct igb_vf *vf)
{
/*
* VTCTRL.RST clears the VF's queue-enable and interrupt registers
* (I350 section 8.28.1). It therefore also removes the admin-vector
* route needed to deliver the reset notification below. MDD recovery
* explicitly permits toggling VFTE instead (section 7.8.3.8.3).
*
* Leave the interrupt registers intact, keep VFTE/VFRE disabled until
* the VF completes a new reset handshake, and use the no-CTS control
* message to make the guest reinitialize. FreeBSD and DPDK consume
* that message directly; Linux ACKs it and the PF's non-CTS ACK path
* replies with the NACK that schedules igbvf's reset task.
*
* Sanitization failure leaves the pool disabled. The VF reset
* handshake retries it and is NACKed while a queue remains active.
*/
(void)igb_iov_reset_event_common(sc, vf, false);
}
static void
igb_iov_reset_msg(struct e1000_softc *sc, struct igb_vf *vf)
{
struct e1000_hw *hw;
u32 msg[3], reg;
hw = &sc->hw;
if (!igb_iov_reset_event(sc, vf)) {
msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
e1000_write_mbx(hw, msg, 1, vf->pool);
return;
}
igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
igb_iov_set_anti_spoof(sc, vf);
reg = E1000_READ_REG(hw, E1000_VFTE);
E1000_WRITE_REG(hw, E1000_VFTE, reg | (1U << vf->pool));
reg = E1000_READ_REG(hw, E1000_VFRE);
E1000_WRITE_REG(hw, E1000_VFRE, reg | (1U << vf->pool));
/*
* 82576's WVBR blocked bitmap is read-clear, so the reset handshake
* completes that event's lifetime. I350 MDFB might be read-only;
* re-arm its edge latch only after a valid MDFB sample reads clear.
*/
if (hw->mac.type == e1000_82576)
vf->flags &= ~IGB_VF_MDD_BLOCKED;
vf->flags |= IGB_VF_CTS;
memset(msg, 0, sizeof(msg));
msg[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
memcpy(&msg[1], vf->mac, ETHER_ADDR_LEN);
e1000_write_mbx(hw, msg, 3, vf->pool);
}
static int
igb_iov_set_mac_filter(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
{
struct igb_vf_mac_filter *filter, *free_filter;
const u8 *mac;
u32 info;
int count, i;
info = msg[0] & E1000_VT_MSGINFO_MASK;
if (info == E1000_VF_MAC_FILTER_CLR) {
igb_iov_clear_mac_filters(sc, vf);
return (0);
}
if (info != E1000_VF_MAC_FILTER_ADD)
return (EINVAL);
if ((vf->flags & IGB_VF_CAP_MAC) == 0)
return (EPERM);
mac = (const u8 *)&msg[1];
if (!igb_iov_mac_valid(mac))
return (EINVAL);
if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) == 0)
return (0);
count = 0;
free_filter = NULL;
for (i = 0; i < sc->num_vf_mac_filters; i++) {
filter = &sc->vf_mac_filters[i];
if (!filter->active) {
if (free_filter == NULL)
free_filter = filter;
continue;
}
if (memcmp(filter->mac, mac, ETHER_ADDR_LEN) != 0)
continue;
return (filter->pool == vf->pool ? 0 : EADDRINUSE);
}
for (i = 0; i < sc->num_vf_mac_filters; i++)
if (sc->vf_mac_filters[i].active &&
sc->vf_mac_filters[i].pool == vf->pool)
count++;
if (igb_iov_mac_in_use(sc, mac, vf))
return (EADDRINUSE);
if (count >= IGB_IOV_MAX_MAC_FILTERS)
return (ENOSPC);
if (free_filter == NULL)
return (ENOSPC);
free_filter->active = true;
free_filter->pool = vf->pool;
memcpy(free_filter->mac, mac, ETHER_ADDR_LEN);
igb_iov_map_rar(sc, free_filter->rar_index, free_filter->mac, vf->pool);
return (0);
}
static int
igb_iov_set_mac(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
{
u8 *mac;
if ((msg[0] & E1000_VT_MSGINFO_MASK) != 0)
return (igb_iov_set_mac_filter(sc, vf, msg));
mac = (u8 *)&msg[1];
if (!igb_iov_mac_valid(mac))
return (EINVAL);
if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
!(vf->flags & IGB_VF_CAP_MAC))
return (EPERM);
if (memcmp(mac, vf->mac, ETHER_ADDR_LEN) != 0 &&
igb_iov_mac_in_use(sc, mac, vf))
return (EADDRINUSE);
memcpy(vf->mac, mac, ETHER_ADDR_LEN);
igb_iov_map_rar(sc, vf->rar_index, vf->mac, vf->pool);
return (0);
}
static int
igb_iov_set_multicast(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
{
u16 hashes[IGB_IOV_MAX_MC_HASHES] = {};
bool overflow;
int count, i;
count = (msg[0] & E1000_VF_SET_MULTICAST_COUNT_MASK) >>
E1000_VT_MSGINFO_SHIFT;
overflow = count > IGB_IOV_MAX_MC_HASHES ||
(msg[0] & E1000_VF_SET_MULTICAST_OVERFLOW) != 0;
count = min(count, IGB_IOV_MAX_MC_HASHES);
for (i = 0; i < count; i++)
hashes[i] =
(msg[1 + i / 2] >> ((i & 1) * 16)) & 0xffff;
if (vf->mc_count == count &&
((vf->flags & IGB_VF_MCAST_OVERFLOW) != 0) == overflow &&
memcmp(vf->mc_hashes, hashes, sizeof(hashes)) == 0)
return (0);
memcpy(vf->mc_hashes, hashes, sizeof(vf->mc_hashes));
vf->mc_count = count;
if (overflow)
vf->flags |= IGB_VF_MCAST_OVERFLOW;
else
vf->flags &= ~IGB_VF_MCAST_OVERFLOW;
if (overflow &&
(vf->flags & IGB_VF_MCAST_OVERFLOW_WARNED) == 0) {
vf->flags |= IGB_VF_MCAST_OVERFLOW_WARNED;
device_printf(sc->dev,
"VF %u multicast list exceeds 30 entries; "
"enabling all-multicast reception\n", vf->pool);
}
igb_iov_configure_vmolr(sc, vf);
igb_iov_rebuild_mta(sc);
return (0);
}
static int
igb_iov_set_lpe(struct e1000_softc *sc, struct igb_vf *vf, u32 *msg)
{
u32 size;
size = msg[1];
if (size < ETHER_MIN_LEN)
return (EINVAL);
vf->max_frame_size = min(size, IGB_IOV_MAX_FRAME_SIZE);
igb_iov_configure_vmolr(sc, vf);
return (0);
}
static int
igb_iov_set_promisc(struct e1000_softc *sc, struct igb_vf *vf, u32 msg)
{
u32 mode;
mode = msg & E1000_VT_MSGINFO_MASK;
if (mode & ~(E1000_VF_SET_PROMISC_UNICAST |
E1000_VF_SET_PROMISC_MULTICAST))
return (EINVAL);
if (mode != 0 && !(vf->flags & IGB_VF_ALLOW_PROMISC))
return (EPERM);
vf->flags &= ~(IGB_VF_UCAST_PROMISC | IGB_VF_MCAST_PROMISC);
if (mode & E1000_VF_SET_PROMISC_UNICAST)
vf->flags |= IGB_VF_UCAST_PROMISC;
if (mode & E1000_VF_SET_PROMISC_MULTICAST)
vf->flags |= IGB_VF_MCAST_PROMISC;
igb_iov_configure_vmolr(sc, vf);
igb_iov_set_uta(sc);
return (0);
}
static bool
igb_iov_process_msg(struct e1000_softc *sc, struct igb_vf *vf)
{
struct e1000_hw *hw;
u32 msg[E1000_VFMAILBOX_SIZE], type;
int error;
hw = &sc->hw;
memset(msg, 0, sizeof(msg));
if (e1000_read_mbx(hw, msg, nitems(msg), vf->pool, false) != 0)
return (false);
vf->flags &= ~IGB_VF_MBX_PENDING;
vf->mbx_retry_at = 0;
vf->mbx_retry_count = 0;
if (msg[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) {
e1000_unlock_mbx(hw, vf->pool);
return (true);
}
if (msg[0] == E1000_VF_RESET) {
igb_iov_reset_msg(sc, vf);
return (true);
}
if (!(vf->flags & IGB_VF_CTS)) {
if (igb_iov_nack_allowed(vf)) {
msg[0] = igb_iov_reply_header(msg[0], false, false);
e1000_write_mbx(hw, msg, 1, vf->pool);
} else
e1000_unlock_mbx(hw, vf->pool);
return (true);
}
type = msg[0] & 0xffff;
switch (type) {
case E1000_VF_SET_MAC_ADDR:
error = igb_iov_set_mac(sc, vf, msg);
break;
case E1000_VF_SET_MULTICAST:
error = igb_iov_set_multicast(sc, vf, msg);
break;
case E1000_VF_SET_VLAN:
if (vf->default_vlan != 0)
error = EPERM;
else if ((msg[1] & ~E1000_VLVF_VLANID_MASK) != 0)
error = EINVAL;
else
error = igb_iov_set_vlan(sc, vf,
msg[1] & E1000_VLVF_VLANID_MASK,
(msg[0] & E1000_VF_SET_VLAN_ADD) != 0);
break;
case E1000_VF_SET_LPE:
error = igb_iov_set_lpe(sc, vf, msg);
break;
case E1000_VF_SET_PROMISC:
error = igb_iov_set_promisc(sc, vf, msg[0]);
break;
default:
error = EOPNOTSUPP;
break;
}
msg[0] = igb_iov_reply_header(msg[0], true, error == 0);
e1000_write_mbx(hw, msg, 1, vf->pool);
return (true);
}
static sbintime_t
igb_iov_service_pending_mbx(struct e1000_softc *sc, struct igb_vf *vf,
sbintime_t now)
{
sbintime_t delay;
if ((vf->flags & IGB_VF_MBX_PENDING) == 0)
return (0);
if (vf->mbx_retry_at != 0 && now < vf->mbx_retry_at)
return (vf->mbx_retry_at);
if (igb_iov_process_msg(sc, vf))
return (0);
now = getsbinuptime();
if (vf->mbx_retry_count < IGB_IOV_MBX_RETRY_COUNT) {
delay = igb_iov_mbx_retry_delay[vf->mbx_retry_count++];
vf->mbx_retry_at = now + delay;
return (vf->mbx_retry_at);
}
vf->flags &= ~(IGB_VF_CTS | IGB_VF_MBX_PENDING);
vf->flags |= IGB_VF_MBX_GAVE_UP;
vf->mbx_retry_at = 0;
if (ratecheck(&vf->last_mbx_log, &igb_iov_mbx_log_interval))
device_printf(sc->dev,
"mailbox remained busy for VF %u; CTS revoked\n",
vf->pool);
return (0);
}
void
igb_iov_handle_mbx(struct e1000_softc *sc)
{
struct e1000_hw *hw;
struct igb_vf *vf;
sbintime_t delay, next_retry_at, now, retry_at;
u32 msg;
int i;
if (!sc->iov_hw_active)
return;
hw = &sc->hw;
next_retry_at = 0;
for (i = 0; i < sc->num_vfs; i++) {
vf = &sc->vfs[i];
if (!(vf->flags & IGB_VF_ACTIVE))
continue;
now = getsbinuptime();
if (e1000_check_for_rst(hw, vf->pool) == 0) {
/*
* The old VF is gone. A new owner's reset handshake
* reruns sanitization before enabling its pool.
*/
(void)igb_iov_reset_event(sc, vf);
}
if ((vf->flags &
(IGB_VF_MBX_PENDING | IGB_VF_MBX_GAVE_UP)) == 0 &&
e1000_check_for_msg(hw, vf->pool) == 0) {
vf->flags |= IGB_VF_MBX_PENDING;
vf->mbx_retry_at = 0;
vf->mbx_retry_count = 0;
}
retry_at = igb_iov_service_pending_mbx(sc, vf, now);
if (retry_at != 0 &&
(next_retry_at == 0 || retry_at < next_retry_at))
next_retry_at = retry_at;
if (e1000_check_for_ack(hw, vf->pool) == 0 &&
!(vf->flags & IGB_VF_CTS) && igb_iov_nack_allowed(vf)) {
msg = E1000_VT_MSGTYPE_NACK;
e1000_write_mbx(hw, &msg, 1, vf->pool);
}
}
if (next_retry_at != 0) {
delay = next_retry_at - getsbinuptime();
if (delay <= 0)
delay = SBT_1MS;
callout_reset_sbt(&sc->iov_mbx_retry, delay, 0,
igb_iov_mbx_retry_callout, sc, C_PREL(1));
}
}
static bool
igb_iov_notify_vf_mdd_reset(struct e1000_softc *sc, struct igb_vf *vf)
{
u32 msg;
/*
* MDD recovery preserves the VF's admin-vector configuration. Send
* the same no-CTS control message used for PF reset notification so
* the VF discards its state and completes a new reset handshake.
* A failed write is retried from the timer-driven admin pass; the VF's
* transmit watchdog remains the final fallback when traffic is still
* queued and notification never succeeds.
*/
msg = E1000_PF_CONTROL_MSG;
if (e1000_write_mbx(&sc->hw, &msg, 1, vf->pool) != 0) {
vf->mdd_notify_at =
getsbinuptime() + igb_iov_mdd_notify_retry;
if (ratecheck(&vf->last_mbx_log,
&igb_iov_mbx_log_interval))
device_printf(sc->dev,
"could not notify VF %u of malicious-driver "
"reset; will retry\n", vf->pool);
return (false);
}
vf->flags &= ~IGB_VF_MDD_NOTIFY_PENDING;
vf->mdd_notify_at = 0;
return (true);
}
void
igb_iov_handle_mdd(struct e1000_softc *sc)
{
struct igb_vf *vf;
u32 blocked, cleared, handled, lvmmc;
u32 readback, spoofed;
bool mdfb_valid, pending;
int i;
pending = atomic_readandclear_32(&sc->iov_pending) != 0;
lvmmc = pending ?
atomic_readandclear_32(&sc->iov_mdd_cause) : 0;
if (!sc->iov_hw_active) {
atomic_readandclear_32(&sc->iov_spoof_pending);
atomic_readandclear_32(&sc->iov_blocked_pending);
return;
}
blocked = 0;
handled = 0;
mdfb_valid = false;
if (sc->hw.mac.type == e1000_i350) {
u32 mdfb;
spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
/*
* I350 reports ordinary MAC/VLAN spoofing through the
* interrupt-time LVMMC snapshot rather than WVBR. The
* filter accumulates Last_Q into iov_spoof_pending so events
* from different VFs coalesce safely until this timer-driven
* admin pass.
*/
spoofed &= IGB_I350_QUEUE_MASK;
/*
* Sample MDFB on every admin pass so a blocked queue is not
* mislabeled as an ordinary spoof when no MDDET observation
* is pending.
*/
mdfb = E1000_READ_REG(&sc->hw, E1000_MDFB);
if (__predict_false(mdfb == 0xffffffff))
mdfb = 0;
else {
mdfb &= IGB_I350_QUEUE_MASK;
mdfb_valid = true;
}
/*
* I350 SDM sections 8.14.10 and 8.14.11: WVBR reports
* spoof and malicious-driver events, while MDFB identifies
* the queues actually blocked for malicious behavior.
*/
spoofed &= ~mdfb;
blocked = mdfb;
if (blocked != 0 && lvmmc == 0)
lvmmc = E1000_READ_REG(&sc->hw, E1000_LVMMC);
/*
* A failed diagnostic read does not invalidate the
* blocked-queue bitmap that was read successfully above.
*/
if (__predict_false(lvmmc == 0xffffffff))
lvmmc = 0;
/*
* MDFB is authoritative for queues stopped by malicious-driver
* detection. LVMMC reports causes such as VLAN IERR and
* Mal_PF, but its Last_Q field does not establish that a queue
* was blocked. Do not manufacture a blocked bit when MDFB is
* clear.
*/
} else {
if (!pending)
return;
/*
* WVBR is read-clear and does not preserve every queue across
* multiple MDDET interrupts. The interrupt filter snapshots and
* accumulates its pool bitmaps before this deferred admin pass.
*/
spoofed = atomic_readandclear_32(&sc->iov_spoof_pending);
blocked = atomic_readandclear_32(&sc->iov_blocked_pending);
/* A blocked-queue classification dominates its WVBR low bit. */
spoofed &= ~blocked;
}
for (i = 0; i < sc->num_vfs; i++) {
vf = &sc->vfs[i];
if (!(vf->flags & IGB_VF_ACTIVE))
continue;
if ((vf->flags & IGB_VF_MDD_NOTIFY_PENDING) != 0 &&
getsbinuptime() >= vf->mdd_notify_at)
(void)igb_iov_notify_vf_mdd_reset(sc, vf);
/*
* An invalid MDFB sample must neither report a new edge nor
* masquerade as evidence that an old edge has cleared.
*/
if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
(blocked & (1U << i)) == 0)
vf->flags &= ~IGB_VF_MDD_BLOCKED;
if ((spoofed & (1U << i)) != 0 &&
ratecheck(&vf->last_spoof_log,
&igb_iov_spoof_log_interval))
device_printf(sc->dev,
"spoof event detected from VF %u; packet dropped\n",
vf->pool);
if ((blocked & (1U << i)) == 0)
continue;
if ((vf->flags & IGB_VF_MDD_BLOCKED) != 0)
continue;
vf->flags |= IGB_VF_MDD_BLOCKED;
if (ratecheck(&vf->last_mdd_log, &igb_iov_mdd_log_interval))
device_printf(sc->dev,
"malicious-driver event 0x%08x from VF %u; "
"resetting VF\n", lvmmc, vf->pool);
igb_iov_mdd_reset_event(sc, vf);
vf->flags |= IGB_VF_MDD_NOTIFY_PENDING;
(void)igb_iov_notify_vf_mdd_reset(sc, vf);
handled |= 1U << i;
}
if (sc->hw.mac.type == e1000_i350 && mdfb_valid &&
(blocked & (1U << sc->pool)) == 0)
sc->iov_pf_mdd_blocked = false;
if ((blocked & (1U << sc->pool)) != 0 &&
(sc->hw.mac.type != e1000_i350 ||
!sc->iov_pf_mdd_blocked)) {
if (sc->hw.mac.type == e1000_i350)
sc->iov_pf_mdd_blocked = true;
if (ratecheck(&sc->iov_last_mdd_log,
&igb_iov_mdd_log_interval))
device_printf(sc->dev,
"malicious-driver event 0x%08x from PF queue; "
"resetting PF\n", lvmmc);
iflib_request_reset(sc->ctx);
iflib_admin_intr_deferred(sc->ctx);
handled |= 1U << sc->pool;
}
if (sc->hw.mac.type == e1000_i350 && handled != 0) {
/*
* I350 documentation conflicts: the register summary calls
* MDFB RWS while the detailed field table calls it RO. I350
* silicon clears a blocked bit when software writes it back.
* Write only bits whose recovery was initiated. If a revision
* instead implements MDFB as RO, the edge latch above prevents
* a reset loop and this one transition-time write is harmless.
*/
E1000_WRITE_REG(&sc->hw, E1000_MDFB, handled);
E1000_WRITE_FLUSH(&sc->hw);
/*
* Rearm from observed hardware state instead of waiting for
* the next admin pass. The PF context lock prevents a reset
* handshake from re-enabling the VF before this readback. A
* write-to-clear part reports zero; a read-only part retains
* the bit and therefore retains the one-shot edge latch.
*/
readback = E1000_READ_REG(&sc->hw, E1000_MDFB);
if (__predict_false(readback == 0xffffffff))
cleared = 0;
else
cleared = handled &
~(readback & IGB_I350_QUEUE_MASK);
for (i = 0; i < sc->num_vfs; i++)
if ((cleared & (1U << i)) != 0)
sc->vfs[i].flags &= ~IGB_VF_MDD_BLOCKED;
if ((cleared & (1U << sc->pool)) != 0)
sc->iov_pf_mdd_blocked = false;
}
if (sc->hw.mac.type == e1000_i350) {
/*
* I350 can retain EICR.OTHER without delivering the admin MSI-X
* even though its EIMS and legacy IMS bits remain enabled. Kick
* the already-enabled vector on each admin pass so its filter
* consumes any retained ICR/LVMMC cause. A synthetic interrupt
* with no legacy cause is handled entirely by the filter.
*/
E1000_WRITE_REG(&sc->hw, E1000_EICS, sc->link_mask);
E1000_WRITE_FLUSH(&sc->hw);
}
}
void
igb_iov_mdd_event(struct e1000_softc *sc)
{
u32 blocked, cause, queues, queue, spoofed, wvbr;
/*
* LVMMC is clear-on-read. Preserve it in the interrupt filter, as
* Intel's igb driver does, rather than deferring the only copy.
*/
cause = E1000_READ_REG(&sc->hw, E1000_LVMMC);
if (__predict_false(cause == 0xffffffff))
return;
if (sc->hw.mac.type == e1000_82576) {
/*
* Snapshot WVBR in the interrupt filter. Waiting for the admin
* task loses all but the last of back-to-back VF MDD events on
* 82576. Convert the staggered queue map into pool bits and OR
* each observation into software latches for deferred recovery.
*/
wvbr = E1000_READ_REG(&sc->hw, E1000_WVBR);
if (__predict_false(wvbr == 0xffffffff)) {
spoofed = 0;
blocked = 0;
} else {
queues = wvbr & IGB_82576_QUEUE_MASK;
spoofed = (queues & 0xff) |
(queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
queues = (wvbr >> 16) & IGB_82576_QUEUE_MASK;
blocked = (queues & 0xff) |
(queues >> IGB_82576_STAGGERED_QUEUE_SHIFT);
}
SDT_PROBE4(igb_iov, mdd, sample, wvbr, sc, wvbr, spoofed,
blocked);
/*
* 82576 can report a coalesced block-class event with all affected
* queues in WVBR's low half and no high-half blocked bits. If an
* ordinary spoof shares that snapshot, the register has no per-queue
* cause information. Deliberately fail closed by recovering every
* low-half queue; this can reset a spoof-only sibling, but avoids
* stranding a blocked VF. LVMMC.Last_Q identifies only the final
* event and lost simultaneous blocked VFs on tested silicon.
*/
if (blocked == 0 &&
(cause & IGB_82576_LVMMC_BLOCK_MASK) != 0) {
blocked = spoofed;
if (blocked == 0) {
queue = (cause >> 16) & 0xf;
blocked = 1U << (queue & 0x7);
}
}
if (spoofed != 0)
atomic_set_32(&sc->iov_spoof_pending, spoofed);
if (blocked != 0)
atomic_set_32(&sc->iov_blocked_pending, blocked);
}
if (sc->hw.mac.type == e1000_i350 &&
(cause & IGB_I350_LVMMC_MAC_VLAN_SPOOF) != 0) {
queue = (cause >> IGB_I350_LVMMC_LAST_Q_SHIFT) &
IGB_I350_LVMMC_LAST_Q_MASK;
/*
* FreeBSD assigns one queue to each VF pool, so Last_Q is
* also the VF number. Preserve all VFs observed before the
* timer pass, and do not overwrite an unrelated blocked
* queue's diagnostic with this non-blocking spoof event.
*/
atomic_set_32(&sc->iov_spoof_pending, 1U << queue);
return;
}
atomic_store_rel_32(&sc->iov_mdd_cause, cause);
atomic_set_32(&sc->iov_pending, 1);
}
void
igb_iov_ping_all_vfs(struct e1000_softc *sc)
{
struct igb_vf *vf;
u32 msg;
int i;
if (!sc->iov_hw_active)
return;
for (i = 0; i < sc->num_vfs; i++) {
vf = &sc->vfs[i];
if (!(vf->flags & IGB_VF_ACTIVE))
continue;
msg = E1000_PF_CONTROL_MSG;
if (vf->flags & IGB_VF_CTS)
msg |= E1000_VT_MSGTYPE_CTS;
e1000_write_mbx(&sc->hw, &msg, 1, vf->pool);
}
}
void
igb_iov_initialize(struct e1000_softc *sc)
{
struct e1000_hw *hw;
struct igb_vf *vf;
u32 ctrl_ext, dtxctl, mask, rctl, rplolr, vt_ctl;
int i;
if (sc->num_vfs == 0)
return;
hw = &sc->hw;
atomic_readandclear_32(&sc->iov_mdd_cause);
atomic_readandclear_32(&sc->iov_pending);
atomic_readandclear_32(&sc->iov_spoof_pending);
atomic_readandclear_32(&sc->iov_blocked_pending);
/* Plain VMDq keeps every 82576 PF/VF pool on queue zero. */
E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_ENABLE_VMDQ);
vt_ctl = E1000_READ_REG(hw, E1000_VT_CTL);
vt_ctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
E1000_VT_CTL_DISABLE_DEF_POOL);
vt_ctl |= sc->pool << E1000_VT_CTL_DEFAULT_POOL_SHIFT;
vt_ctl |= E1000_VT_CTL_VM_REPL_EN;
E1000_WRITE_REG(hw, E1000_VT_CTL, vt_ctl);
mask = 1U << sc->pool;
E1000_WRITE_REG(hw, E1000_VFRE, mask);
E1000_WRITE_REG(hw, E1000_VFTE, mask);
/* A VF without RX descriptors must not block any other pool. */
E1000_WRITE_REG(hw, E1000_QDE,
hw->mac.type == e1000_i350 ? IGB_I350_QUEUE_MASK : ALL_QUEUES);
e1000_vmdq_set_loopback_pf(hw, true);
dtxctl = E1000_READ_REG(hw, E1000_DTXCTL);
dtxctl |= E1000_DTXCTL_MDP_EN;
if (hw->mac.type == e1000_82576) {
dtxctl |= E1000_DTXCTL_VLAN_ADDED |
E1000_DTXCTL_SPOOF_INT;
rplolr = E1000_READ_REG(hw, E1000_RPLOLR);
rplolr |= E1000_RPLOLR_STRVLAN;
E1000_WRITE_REG(hw, E1000_RPLOLR, rplolr);
} else {
/*
* I350 SDM section 8.12.5 defines this field with inverted
* polarity: setting it keeps an ordinary spoof from disabling
* the VF queue. Enable its notification as well. I350
* hardware reports the VF in LVMMC.Last_Q (WVBR remains zero);
* the moderated admin vector captures that value, while
* timer-driven administration and per-VF ratecheck bound the
* work and console output.
*/
dtxctl |= E1000_DTXCTL_SPOOF_INT |
IGB_I350_DTXCTL_ENABLE_SPOOF_QUEUE;
}
E1000_WRITE_REG(hw, E1000_DTXCTL, dtxctl);
igb_iov_map_rar(sc, 0, hw->mac.addr, sc->pool);
igb_iov_configure_pf_vmolr(sc);
igb_iov_set_uta(sc);
for (i = 0; i < sc->num_vfs; i++) {
vf = &sc->vfs[i];
if (!(vf->flags & IGB_VF_ACTIVE))
continue;
/*
* A PF-wide reset is trusted and can require a complete guest
* replay. Guest-controlled RESET and VFLR do not refill this
* allowance.
*/
igb_iov_reset_vlan_rate(vf);
igb_iov_clear_mac_filters(sc, vf);
igb_iov_reset_vf_state(sc, vf);
igb_iov_clear_rar(sc, vf->rar_index);
igb_iov_set_anti_spoof(sc, vf);
}
igb_iov_rebuild_mta(sc);
igb_iov_rebuild_vlan(sc);
rctl = E1000_READ_REG(hw, E1000_RCTL);
E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_VFE);
E1000_WRITE_REG(hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
if (hw->mac.type == e1000_i350)
E1000_WRITE_REG(hw, E1000_DMACR, 0);
ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
E1000_WRITE_REG(hw, E1000_CTRL_EXT,
ctrl_ext | E1000_CTRL_EXT_PFRSTD);
E1000_WRITE_FLUSH(hw);
/*
* MDDET remains masked until iov_hw_active is published and iflib
* rearms the admin vector. Programming the per-pool policy above can
* leave a setup-time MDDET observation in the read-clear registers.
* If that stale cause is carried across the unmask, a later ordinary
* spoof can update LVMMC without generating a new interrupt edge.
*
* Drain only after all IOV policy is installed and before exposing the
* active state. Mailbox requests are also serviced by the periodic
* admin pass, and ping_all_vfs() below supplies a fresh notification.
*/
/*
* Clear the setup-time interrupt latch before its diagnostic state.
* I350 does not reliably generate the next MDDET edge when LVMMC is
* consumed while ICR.MDDET remains latched. This differs deliberately
* from the final arm-time drain, where ICR is read last so a later event
* remains pending for the unmask.
*/
if (hw->mac.type == e1000_i350)
(void)E1000_READ_REG(hw, E1000_ICR);
(void)E1000_READ_REG(hw, E1000_LVMMC);
if (hw->mac.type == e1000_82576)
(void)E1000_READ_REG(hw, E1000_WVBR);
if (hw->mac.type != e1000_i350)
(void)E1000_READ_REG(hw, E1000_ICR);
atomic_readandclear_32(&sc->iov_mdd_cause);
atomic_readandclear_32(&sc->iov_pending);
atomic_readandclear_32(&sc->iov_spoof_pending);
atomic_readandclear_32(&sc->iov_blocked_pending);
atomic_store_rel_32(&sc->iov_intr_drain_pending, 1);
sc->iov_hw_active = true;
igb_iov_ping_all_vfs(sc);
}
int
igb_iov_validate(struct e1000_softc *sc, u16 num_vfs)
{
if (!igb_iov_supported(sc))
return (ENXIO);
/* One of the eight hardware pools is reserved for the PF. */
if (num_vfs == 0 || num_vfs > MAX_NUM_VFS)
return (EINVAL);
if (sc->vfs != NULL)
return (EBUSY);
if (sc->intr_type != IFLIB_INTR_MSIX) {
device_printf(sc->dev, "SR-IOV requires MSI-X\n");
return (ENOTSUP);
}
if (sc->tx_num_queues != 1 || sc->rx_num_queues != 1) {
device_printf(sc->dev,
"SR-IOV requires one PF TX and RX queue; set "
"dev.igb.%d.iflib.override_ntxqs=1 and "
"dev.igb.%d.iflib.override_nrxqs=1 before attach\n",
device_get_unit(sc->dev), device_get_unit(sc->dev));
return (EINVAL);
}
return (0);
}
int
igb_if_iov_init(if_ctx_t ctx, u16 num_vfs, const nvlist_t *config)
{
struct e1000_softc *sc;
int error, i;
sc = iflib_get_softc(ctx);
(void)config;
/*
* This callback may run while the PF is down. Record the software
* layout here; igb_iov_initialize() programs it during interface init.
*/
atomic_store_rel_32(&sc->iov_teardown, 0);
error = igb_iov_validate(sc, num_vfs);
if (error != 0)
return (error);
sc->vfs = mallocarray(num_vfs, sizeof(*sc->vfs), M_IGB_IOV,
M_WAITOK | M_ZERO);
sc->num_vf_mac_filters =
sc->hw.mac.rar_entry_count - num_vfs - 1;
sc->vf_mac_filters = mallocarray(sc->num_vf_mac_filters,
sizeof(*sc->vf_mac_filters), M_IGB_IOV, M_WAITOK | M_ZERO);
for (i = 0; i < sc->num_vf_mac_filters; i++)
sc->vf_mac_filters[i].rar_index = i + 1;
sc->pool = num_vfs;
sc->iov_mta_valid = false;
sc->iov_pf_mdd_blocked = false;
sc->tx_queues[0].txr.me = sc->pool;
sc->rx_queues[0].rxr.me = sc->pool;
e1000_init_mbx_params_pf(&sc->hw);
sc->num_vfs = num_vfs;
return (0);
}
void
igb_if_iov_uninit(if_ctx_t ctx)
{
struct e1000_softc *sc;
struct e1000_hw *hw;
u32 mask, rah;
int error, i, iov_pos;
u16 iov_ctl;
sc = iflib_get_softc(ctx);
if (sc->vfs == NULL)
return;
hw = &sc->hw;
sc->iov_hw_active = false;
if (sc->iov_mbx_retry_initialized)
callout_drain(&sc->iov_mbx_retry);
E1000_WRITE_REG(hw, E1000_MBVFIMR, 0);
mask = 1U << sc->pool;
E1000_WRITE_REG(hw, E1000_VFRE, mask);
E1000_WRITE_REG(hw, E1000_VFTE, mask);
/*
* pci_iov(4) invokes the driver before it clears VF Enable. Quiesce
* the VFs and clear it here so that 82576's queue-reuse interval is
* measured from the actual IOV-disable event.
*/
error = pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos);
if (error == 0) {
iov_ctl = pci_read_config(sc->dev,
iov_pos + PCIR_SRIOV_CTL, 2);
iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE);
pci_write_config(sc->dev, iov_pos + PCIR_SRIOV_CTL,
iov_ctl, 2);
if (hw->mac.type == e1000_82576) {
pause("igbiov", MAX(1, howmany(hz, 10)));
E1000_WRITE_REG(hw, E1000_IOVCTL,
E1000_IOVCTL_REUSE_VFQ);
E1000_WRITE_FLUSH(hw);
pause("igbiov", MAX(1, howmany(hz, 10)));
}
} else
device_printf(sc->dev,
"could not disable PCI SR-IOV before queue reuse: %d\n",
error);
E1000_WRITE_REG(hw, E1000_VT_CTL, 0);
e1000_vmdq_set_loopback_pf(hw, false);
e1000_vmdq_set_anti_spoofing_pf(hw, false, 0);
for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++)
E1000_WRITE_REG(hw, E1000_VLVF(i), 0);
for (i = 0; i < sc->num_vfs; i++)
if (sc->vfs[i].flags & IGB_VF_ACTIVE)
igb_iov_clear_rar(sc, sc->vfs[i].rar_index);
for (i = 0; i < sc->num_vf_mac_filters; i++)
if (sc->vf_mac_filters[i].active)
igb_iov_clear_rar(sc, sc->vf_mac_filters[i].rar_index);
rah = E1000_READ_REG(hw, E1000_RAH(0));
rah &= ~IGB_IOV_RAH_POOLSEL_MASK;
E1000_WRITE_REG(hw, E1000_RAH(0), rah);
free(sc->vfs, M_IGB_IOV);
free(sc->vf_mac_filters, M_IGB_IOV);
sc->vfs = NULL;
sc->vf_mac_filters = NULL;
sc->num_vfs = 0;
sc->num_vf_mac_filters = 0;
sc->pool = 0;
sc->iov_mta_valid = false;
sc->iov_pf_mdd_blocked = false;
sc->iov_pf_vlan_promisc = false;
igb_iov_vfta_shadow_invalidate(sc);
sc->tx_queues[0].txr.me = 0;
sc->rx_queues[0].rxr.me = 0;
atomic_readandclear_32(&sc->iov_mdd_cause);
atomic_readandclear_32(&sc->iov_pending);
atomic_readandclear_32(&sc->iov_spoof_pending);
atomic_readandclear_32(&sc->iov_blocked_pending);
atomic_readandclear_32(&sc->iov_intr_drain_pending);
atomic_store_rel_32(&sc->iov_teardown, 0);
}
static bool
igb_iov_mac_in_use(struct e1000_softc *sc, const u8 *mac,
const struct igb_vf *skip)
{
int i;
if (memcmp(sc->hw.mac.addr, mac, ETHER_ADDR_LEN) == 0)
return (true);
for (i = 0; i < sc->num_vfs; i++)
if (&sc->vfs[i] != skip &&
(sc->vfs[i].flags & IGB_VF_ACTIVE) != 0 &&
memcmp(sc->vfs[i].mac, mac, ETHER_ADDR_LEN) == 0)
return (true);
for (i = 0; i < sc->num_vf_mac_filters; i++)
if (sc->vf_mac_filters[i].active &&
memcmp(sc->vf_mac_filters[i].mac, mac,
ETHER_ADDR_LEN) == 0)
return (true);
return (false);
}
int
igb_if_iov_vf_add(if_ctx_t ctx, u16 vfnum, const nvlist_t *config)
{
struct e1000_softc *sc;
struct igb_vf *vf;
struct ether_addr generated;
const void *mac;
char nameunit[IFNAMSIZ + sizeof("-vf65535")];
size_t mac_size;
uint64_t configured_vlan;
u16 vlan;
sc = iflib_get_softc(ctx);
if (vfnum >= sc->num_vfs)
return (EINVAL);
vf = &sc->vfs[vfnum];
if (vf->flags & IGB_VF_ACTIVE)
return (EBUSY);
configured_vlan = nvlist_get_number(config, "vlan");
if (configured_vlan > VF_VLAN_TRUNK)
return (EINVAL);
vlan = configured_vlan;
if (vlan == 0)
return (ENOTSUP);
if (vlan == VF_VLAN_TRUNK)
vlan = 0;
if (!igb_iov_vlan_present(sc, vlan, false) &&
igb_iov_vlan_unique_count(sc, false) >=
E1000_VLVF_ARRAY_SIZE)
return (ENOSPC);
vf->pool = vfnum;
vf->rar_index = sc->hw.mac.rar_entry_count - (vfnum + 1);
vf->max_frame_size = ETHER_MAX_LEN;
vf->default_vlan = vlan;
igb_iov_reset_vlan_rate(vf);
if (nvlist_exists_binary(config, "mac-addr")) {
mac = nvlist_get_binary(config, "mac-addr", &mac_size);
if (mac_size != ETHER_ADDR_LEN || !igb_iov_mac_valid(mac))
return (EINVAL);
if (igb_iov_mac_in_use(sc, mac, vf))
return (EADDRINUSE);
memcpy(vf->mac, mac, ETHER_ADDR_LEN);
} else {
snprintf(nameunit, sizeof(nameunit), "%s-vf%u",
device_get_nameunit(sc->dev), vfnum);
ether_gen_addr_byname(nameunit, &generated);
memcpy(vf->mac, generated.octet, ETHER_ADDR_LEN);
if (igb_iov_mac_in_use(sc, vf->mac, vf))
return (EADDRINUSE);
}
if (nvlist_get_bool(config, "allow-set-mac"))
vf->flags |= IGB_VF_CAP_MAC;
if (nvlist_get_bool(config, "mac-anti-spoof"))
vf->flags |= IGB_VF_MAC_ANTI_SPOOF;
if (nvlist_get_bool(config, "allow-promisc"))
vf->flags |= IGB_VF_ALLOW_PROMISC;
vf->flags |= IGB_VF_ACTIVE;
igb_iov_reset_vf_state(sc, vf);
igb_iov_set_anti_spoof(sc, vf);
igb_iov_rebuild_vlan(sc);
E1000_WRITE_REG(&sc->hw, E1000_MBVFIMR, igb_iov_active_mask(sc));
return (0);
}
#endif /* PCI_IOV */
Reference in new issue View Git Blame Copy Permalink