iovctl: Report SR-IOV status

Add -L to query the generic packed-nvlist IOV_GET_STATUS interface.
Report PF enable state and configured and total VF counts.  For each VF,
print its PCI address, newbus attachment, bound driver, and ppt state.

Retry size negotiation if the topology changes between ioctls and reject
malformed or incompatible status records.

Keep NIC-specific operational state in ifconfig -v; iovctl owns the
device-neutral PCI topology and applies to any SR-IOV device class.

Relnotes:	yes

sbin/ifconfig/ifconfig.8

@@ -376,7 +376,9 @@ virtual functions: 1
 .Pp
 See
 .Xr iovctl.conf 5
-for VF configuration.
+for VF configuration and
+.Xr iovctl 8
+for PCI attachment and passthrough status.
 .It Ar address
 For the inet family,
 the address is either a host name present in the host name data

usr.sbin/iovctl/iovctl.8

@@ -39,12 +39,15 @@
 .Op Fl f Ar config-file | Fl d Ar device
 .Op Fl n
 .Nm
+.Fl L
+.Op Fl f Ar config-file | Fl d Ar device
+.Nm
 .Fl S
 .Op Fl f Ar config-file | Fl d Ar device
 .Sh DESCRIPTION
 The
 .Nm
-utility creates or destroys PCI Single-Root I/O Virtualization
+utility configures and inspects PCI Single-Root I/O Virtualization
 .Pq SR-IOV
 Virtual Functions
 .Pq VFs .
@@ -89,7 +92,8 @@ For the
 .Fl C
 option, this file will be used to specify all configuration values.
 For the
-.Fl D
+.Fl D ,
+.Fl L ,
 and
 .Fl S
 options, this file will only be used to specify the name of the PF device.
@@ -105,8 +109,22 @@ Perform a dry-run.
 Perform all validation of the specified action and print what would be done,
 but do not perform the actual creation or destruction of VFs.
 This option may not be used with the
+.Fl L
+or
 .Fl S
 flag.
+.It Fl L
+Report the current PCI SR-IOV topology of the specified PF.
+The output identifies the PF and each configured VF by PCI location, reports
+whether each VF is attached to a host driver, and identifies bindings to
+.Va ppt .
+An unconfigured PF reports its enable state and VF capacity without any
+per-VF records.
+A passthrough binding does not imply that a running virtual machine currently
+owns the VF.
+See
+.Xr vmm 4
+for passthrough-device configuration.
 .It Fl S
 Read the configuration schema from the specified device and print its contents
 to stdout.
@@ -114,13 +132,14 @@ This action may be used to discover the configuration parameters supported on
 a given PF device.
 .El
 .Pp
-After creating VFs, use
+For network devices, use
 .Xr ifconfig 8
 with the
 .Fl v
 option on the PF interface to display NIC-specific VF initialization,
 resource, and policy state.
 .Sh SEE ALSO
+.Xr vmm 4 ,
 .Xr iovctl.conf 5 ,
 .Xr rc.conf 5 ,
 .Xr ifconfig 8

usr.sbin/iovctl/iovctl.c

@@ -43,6 +43,7 @@
 static void	config_action(const char *filename, int dryrun);
 static void	delete_action(const char *device, int dryrun);
 static void	print_schema(const char *device);
+static void	print_status(const char *device);
 
 /*
  * Fetch the config schema from the kernel via ioctl.  This function has to
@@ -85,6 +86,49 @@ get_schema(int fd)
 	return (schema);
 }
 
+/* Fetch and unpack the current PCI SR-IOV status. */
+static nvlist_t *
+get_status(int fd)
+{
+	struct pci_iov_status arg;
+	nvlist_t *status;
+	void *buf, *newbuf;
+	size_t buflen;
+	int error;
+
+	buf = NULL;
+	buflen = 0;
+	for (;;) {
+		memset(&arg, 0, sizeof(arg));
+		arg.status = (uintptr_t)buf;
+		arg.len = buflen;
+		error = ioctl(fd, IOV_GET_STATUS, &arg);
+		if (error != 0)
+			err(1, "Could not fetch SR-IOV status");
+		if (arg.error == 0)
+			break;
+		if (arg.error != EMSGSIZE || arg.len <= buflen ||
+		    arg.len > SIZE_MAX) {
+			errno = arg.error;
+			err(1, "Could not fetch SR-IOV status");
+		}
+		newbuf = realloc(buf, arg.len);
+		if (newbuf == NULL)
+			err(1, "Could not allocate %zu bytes for SR-IOV status",
+			    arg.len);
+		buf = newbuf;
+		buflen = arg.len;
+	}
+	if (arg.len == 0 || arg.len > buflen)
+		errx(1, "Kernel returned an invalid SR-IOV status length");
+
+	status = nvlist_unpack(buf, arg.len, 0);
+	if (status == NULL)
+		err(1, "Could not unpack SR-IOV status");
+	free(buf);
+	return (status);
+}
+
 /*
  * Call the ioctl that activates SR-IOV and creates the VFs.
  */
@@ -155,6 +199,7 @@ usage(void)
 
 	warnx("Usage: iovctl -C -f <config file> [-n]");
 	warnx("       iovctl -D [-d <PF device> | -f <config file>] [-n]");
+	warnx("       iovctl -L [-d <PF device> | -f <config file>]");
 	warnx("       iovctl -S [-d <PF device> | -f <config file>]");
 	exit(1);
 
@@ -164,6 +209,7 @@ enum main_action {
 	NONE,
 	CONFIG,
 	DELETE,
+	PRINT_STATUS,
 	PRINT_SCHEMA,
 };
 
@@ -180,12 +226,11 @@ main(int argc, char **argv)
 	dryrun = 0;
 	action = NONE;
 
-	while ((ch = getopt(argc, argv, "Cd:Df:nS")) != -1) {
+	while ((ch = getopt(argc, argv, "Cd:Df:LnS")) != -1) {
 		switch (ch) {
 		case 'C':
 			if (action != NONE) {
-				warnx(
-				   "Only one of -C, -D or -S may be specified");
+				warnx("Only one action may be specified");
 				usage();
 			}
 			action = CONFIG;
@@ -195,8 +240,7 @@ main(int argc, char **argv)
 			break;
 		case 'D':
 			if (action != NONE) {
-				warnx(
-				   "Only one of -C, -D or -S may be specified");
+				warnx("Only one action may be specified");
 				usage();
 			}
 			action = DELETE;
@@ -207,10 +251,16 @@ main(int argc, char **argv)
 		case 'n':
 			dryrun = 1;
 			break;
+		case 'L':
+			if (action != NONE) {
+				warnx("Only one action may be specified");
+				usage();
+			}
+			action = PRINT_STATUS;
+			break;
 		case 'S':
 			if (action != NONE) {
-				warnx(
-				   "Only one of -C, -D or -S may be specified");
+				warnx("Only one action may be specified");
 				usage();
 			}
 			action = PRINT_SCHEMA;
@@ -260,6 +310,16 @@ main(int argc, char **argv)
 		print_schema(device);
 		free(device);
 		break;
+	case PRINT_STATUS:
+		if (dryrun) {
+			warnx("-n flag cannot be used with the -L flag");
+			usage();
+		}
+		if (device == NULL)
+			device = find_device(filename);
+		print_status(device);
+		free(device);
+		break;
 	default:
 		usage();
 		break;
@@ -312,6 +372,106 @@ delete_action(const char *dev_name, int dryrun)
 	close(fd);
 }
 
+static void
+validate_status(const nvlist_t *status)
+{
+	const nvlist_t *pf;
+	const nvlist_t * const *vfs;
+	size_t i, num_vfs;
+	uint64_t configured_vfs, index, total_vfs, version;
+
+	if (!nvlist_exists_number(status, IOV_STATUS_VERSION_NAME) ||
+	    !nvlist_exists_nvlist(status, IOV_STATUS_PF_NAME))
+		errx(1, "Kernel returned an invalid SR-IOV status");
+	version = nvlist_get_number(status, IOV_STATUS_VERSION_NAME);
+	if (version != IOV_STATUS_VERSION)
+		errx(1, "Unsupported SR-IOV status version %ju",
+		    (uintmax_t)version);
+
+	pf = nvlist_get_nvlist(status, IOV_STATUS_PF_NAME);
+	if (!nvlist_exists_string(pf, IOV_STATUS_DEVICE_NAME) ||
+	    !nvlist_exists_string(pf, IOV_STATUS_PCI_LOCATION_NAME) ||
+	    !nvlist_exists_bool(pf, IOV_STATUS_ENABLED_NAME) ||
+	    !nvlist_exists_number(pf, IOV_STATUS_NUM_VFS_NAME) ||
+	    !nvlist_exists_number(pf, IOV_STATUS_TOTAL_VFS_NAME))
+		errx(1, "Kernel returned an invalid SR-IOV PF status");
+	configured_vfs = nvlist_get_number(pf, IOV_STATUS_NUM_VFS_NAME);
+	total_vfs = nvlist_get_number(pf, IOV_STATUS_TOTAL_VFS_NAME);
+	if (configured_vfs > total_vfs)
+		errx(1, "Kernel returned inconsistent SR-IOV VF counts");
+	if (configured_vfs == 0) {
+		if (nvlist_exists(status, IOV_STATUS_VFS_NAME))
+			errx(1, "Kernel returned inconsistent SR-IOV VF counts");
+		return;
+	}
+	if (!nvlist_exists_nvlist_array(status, IOV_STATUS_VFS_NAME))
+		errx(1, "Kernel returned an invalid SR-IOV status");
+	vfs = nvlist_get_nvlist_array(status, IOV_STATUS_VFS_NAME, &num_vfs);
+	if (configured_vfs != num_vfs)
+		errx(1, "Kernel returned inconsistent SR-IOV VF counts");
+	for (i = 0; i < num_vfs; i++) {
+		if (!nvlist_exists_number(vfs[i], IOV_STATUS_VF_INDEX_NAME) ||
+		    !nvlist_exists_string(vfs[i],
+		    IOV_STATUS_PCI_LOCATION_NAME) ||
+		    !nvlist_exists_bool(vfs[i], IOV_STATUS_ATTACHED_NAME) ||
+		    !nvlist_exists_bool(vfs[i], IOV_STATUS_PASSTHROUGH_NAME) ||
+		    (nvlist_exists(vfs[i], IOV_STATUS_BOUND_DRIVER_NAME) &&
+		    !nvlist_exists_string(vfs[i],
+		    IOV_STATUS_BOUND_DRIVER_NAME)))
+			errx(1, "Kernel returned an invalid SR-IOV VF status");
+		index = nvlist_get_number(vfs[i], IOV_STATUS_VF_INDEX_NAME);
+		if (index >= configured_vfs)
+			errx(1, "Kernel returned an invalid SR-IOV VF index");
+	}
+}
+
+static void
+print_status(const char *dev_name)
+{
+	const nvlist_t *pf, *vf;
+	const nvlist_t * const *vfs;
+	nvlist_t *status;
+	size_t i, num_vfs;
+	int fd;
+
+	fd = open_device(dev_name);
+	status = get_status(fd);
+	validate_status(status);
+	pf = nvlist_get_nvlist(status, IOV_STATUS_PF_NAME);
+	vfs = NULL;
+	num_vfs = 0;
+	if (nvlist_exists_nvlist_array(status, IOV_STATUS_VFS_NAME))
+		vfs = nvlist_get_nvlist_array(status, IOV_STATUS_VFS_NAME,
+		    &num_vfs);
+
+	printf("%s:\n", nvlist_get_string(pf, IOV_STATUS_DEVICE_NAME));
+	printf("\tidentity: pci-location=%s\n",
+	    nvlist_get_string(pf, IOV_STATUS_PCI_LOCATION_NAME));
+	printf("\tsriov: enabled=%s vfs=%ju/%ju\n",
+	    nvlist_get_bool(pf, IOV_STATUS_ENABLED_NAME) ? "yes" : "no",
+	    (uintmax_t)nvlist_get_number(pf, IOV_STATUS_NUM_VFS_NAME),
+	    (uintmax_t)nvlist_get_number(pf, IOV_STATUS_TOTAL_VFS_NAME));
+	for (i = 0; i < num_vfs; i++) {
+		vf = vfs[i];
+		printf("\t\tvf %3ju:\n", (uintmax_t)nvlist_get_number(vf,
+		    IOV_STATUS_VF_INDEX_NAME));
+		printf("\t\t\tidentity: pci-location=%s\n",
+		    nvlist_get_string(vf, IOV_STATUS_PCI_LOCATION_NAME));
+		printf("\t\t\thost: attached=%s",
+		    nvlist_get_bool(vf, IOV_STATUS_ATTACHED_NAME) ? "yes" :
+		    "no");
+		if (nvlist_exists_string(vf, IOV_STATUS_BOUND_DRIVER_NAME))
+			printf(" driver=%s", nvlist_get_string(vf,
+			    IOV_STATUS_BOUND_DRIVER_NAME));
+		printf(" passthrough=%s\n",
+		    nvlist_get_bool(vf, IOV_STATUS_PASSTHROUGH_NAME) ? "yes" :
+		    "no");
+	}
+
+	nvlist_destroy(status);
+	close(fd);
+}
+
 static void
 print_default_value(const nvlist_t *parameter, const char *type)
 {

usr.sbin/iovctl/iovctl.conf.5

@@ -141,13 +141,19 @@ The default value of this parameter is false.
 See the PF driver manual page for configuration parameters specific to
 particular hardware.
 .Pp
-For network devices, display VF initialization, resources, and policy state
-with:
+After creating VFs, display their PCI attachment and passthrough state with:
+.Bd -literal -offset indent
+iovctl -L -d ix0
+.Ed
+For network devices, display NIC-specific initialization, resources, and
+policy state with:
 .Bd -literal -offset indent
 ifconfig -v ix0
 .Ed
 See
 .Xr ifconfig 8
+and
+.Xr iovctl 8
 for the meaning of the reported fields.
 .Sh EXAMPLES
 This sample file will create 3 VFs as children of the ix0 device.