mirror of
https://github.com/freebsd/freebsd-src.git
synced 2026-10-28 11:25:21 +01:00
Kevin Bowling
c8343ba84a
bhyve emulates the guest PCI Command register so BAR sizing does not disable physical decoding. However, PCIe Device Control was passed through. A guest VFIO reset therefore performed a physical FLR, which cleared physical Command, while the guest restored only its emulated copy. The device remained assigned with bus mastering disabled and could not fetch DMA descriptors. Intercept guest FLR writes and issue a PPT-managed reset. Stop all vCPUs, verify ownership, quiesce the function, perform only an FLR, and restore the host-owned PCI configuration, decode, and bus-master state. Keep the IOMMU domain in place. bhyve removes guest BAR mappings before this ioctl; a later guest MEMEN write recreates them. Never escalate a guest FLR to a power reset. Reset the guest-owned Command, MSI, MSI-X, MSI-X table, INTx, and MRRS state. PCIe 6.2 section 6.6.2 explicitly preserves MPS across FLR. Virtualize MPS, MRRS, and Completion Timeout. Keep physical MPS and completion-timeout policy host-owned, and apply physical MRRS with MPS as its floor. Keep Phantom Functions Enable host-owned because it changes requester identities visible to the IOMMU. Serialize guest configuration transactions per function and gate trapped and direct BAR access across the reset. Handle byte, word, dword, and overlapping Device Control accesses. A guest FLR can sleep for at least 100 ms. Reserve the target function while dropping the global PPT lock so a guest cannot delay PPT lifecycle operations for other VMs. Operations on the target wait for its reset while other functions and VMs can proceed. Check pcie_flr_supported() before destructive preparation so PPT applies the generic PCI quirk policy. This includes VFs such as the 82599 which implement FLR without advertising it. Validated with two E610 VFs in a Linux 7.0 guest using VFIO no-IOMMU and DPDK testpmd with two queues per VF. Byte, word, dword, and overlapping FLR writes, 32 alternating resets, DPDK traffic, and ixgbevf reattachment all completed while the sibling VF and host PCIe remained healthy. This fixes Linux VFIO no-IOMMU with DPDK PMDs. Reviewed by: markj MFC after: 2 weeks Sponsored by: BBOX.io
77 lines
1.4 KiB
C
77 lines
1.4 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*
|
|
* Copyright (c) 2022 John Baldwin <[email protected]>
|
|
*/
|
|
|
|
#ifndef __VMMAPI_INTERNAL_H__
|
|
#define __VMMAPI_INTERNAL_H__
|
|
|
|
#include <sys/types.h>
|
|
#include <dev/vmm/vmm_mem.h>
|
|
|
|
struct vmctx {
|
|
int fd; /* device file descriptor */
|
|
int ctlfd; /* vmm control descriptor */
|
|
struct {
|
|
vm_paddr_t base;
|
|
vm_size_t size;
|
|
} memsegs[VM_MAX_MEMSEGS];
|
|
size_t lowmem_size;
|
|
size_t highmem_size;
|
|
int memflags;
|
|
char *baseaddr;
|
|
char *name;
|
|
};
|
|
|
|
struct vcpu {
|
|
struct vmctx *ctx;
|
|
int vcpuid;
|
|
};
|
|
|
|
int vcpu_ioctl(struct vcpu *vcpu, u_long cmd, void *arg);
|
|
|
|
extern const char *vm_capstrmap[];
|
|
|
|
#define VM_COMMON_IOCTLS \
|
|
VM_RUN, \
|
|
VM_SUSPEND, \
|
|
VM_REINIT, \
|
|
VM_ALLOC_MEMSEG, \
|
|
VM_GET_MEMSEG, \
|
|
VM_MMAP_MEMSEG, \
|
|
VM_MMAP_MEMSEG, \
|
|
VM_MMAP_GETNEXT, \
|
|
VM_MUNMAP_MEMSEG, \
|
|
VM_SET_REGISTER, \
|
|
VM_GET_REGISTER, \
|
|
VM_SET_REGISTER_SET, \
|
|
VM_GET_REGISTER_SET, \
|
|
VM_INJECT_EXCEPTION, \
|
|
VM_SET_CAPABILITY, \
|
|
VM_GET_CAPABILITY, \
|
|
VM_STATS, \
|
|
VM_STAT_DESC, \
|
|
VM_GLA2GPA_NOFAULT, \
|
|
VM_ACTIVATE_CPU, \
|
|
VM_GET_CPUS, \
|
|
VM_SUSPEND_CPU, \
|
|
VM_RESUME_CPU, \
|
|
VM_SET_TOPOLOGY, \
|
|
VM_GET_TOPOLOGY
|
|
|
|
#define VM_PPT_IOCTLS \
|
|
VM_BIND_PPTDEV, \
|
|
VM_UNBIND_PPTDEV, \
|
|
VM_MAP_PPTDEV_MMIO, \
|
|
VM_PPTDEV_MSI, \
|
|
VM_PPTDEV_MSIX, \
|
|
VM_UNMAP_PPTDEV_MMIO, \
|
|
VM_PPTDEV_DISABLE_MSIX, \
|
|
VM_RESET_PPTDEV
|
|
|
|
extern const cap_ioctl_t vm_ioctl_cmds[];
|
|
extern size_t vm_ioctl_ncmds;
|
|
|
|
#endif /* !__VMMAPI_INTERNAL_H__ */
|