dpaa2: Make software portal holdoff time a sysctl tunable

When debugging the D59463 review, it is very handy to be able
to change the software portal holdoff time without recompiling
the kernel. This commit makes the holdoff time a sysctl tunable,
so it can be changed at runtime.

Tested by:		dsl
Obtained from:		flo_purplekraken.com
MFC after:		3 weeks
Differential Revision:	https://reviews.freebsd.org/D59461
Event:			Berlin Hackathon 202609

sys/dev/dpaa2/dpaa2_io.c

@@ -50,6 +50,7 @@
 #include <sys/mutex.h>
 #include <sys/_cpuset.h>
 #include <sys/cpuset.h>
+#include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 #include <sys/smp.h>
 
@@ -79,6 +80,9 @@
 #define DPIO_IRQ_INDEX		0 /* index of the only DPIO IRQ */
 #define DPIO_POLL_MAX		32
 
+#define SWP_HOLDOFF_MS_MIN	100
+#define SWP_HOLDOFF_MS_MAX	10000
+
 /*
  * Memory:
  *	0: cache-enabled part of the QBMan software portal.
@@ -196,6 +200,51 @@ err_exit:
 	return (error);
 }
 
+static int
+dpaa2_io_collect_irq_holdoff(SYSCTL_HANDLER_ARGS)
+{
+	struct dpaa2_io_softc *sc;
+	struct dpaa2_swp *swp;
+	uint32_t error, new_value;
+
+	sc = (struct dpaa2_io_softc *)arg1;
+	swp = sc->swp;
+	error = SYSCTL_OUT(req, &sc->irq_holdoff,
+	    sizeof(sc->irq_holdoff));
+	if (error || req->newptr == NULL)
+		return error;
+	error = SYSCTL_IN(req, &new_value, sizeof(new_value));
+	if (error)
+		return error;
+	if ((new_value < SWP_HOLDOFF_MS_MIN) || (new_value > SWP_HOLDOFF_MS_MAX))
+		return EINVAL;
+	error = dpaa2_swp_set_irq_coalescing(swp, swp->dqrr.ring_size - 1,
+	    new_value);
+	if (error)
+		return error;
+	sc->irq_holdoff = new_value;
+	return 0;
+}
+
+static void
+dpaa2_io_setup_sysctls(struct dpaa2_io_softc *sc)
+{
+	struct sysctl_ctx_list *ctx;
+	struct sysctl_oid *node;
+	struct sysctl_oid_list *parent;
+
+	ctx = device_get_sysctl_ctx(sc->dev);
+	parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
+
+	node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "io",
+	    CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "io control");
+	parent = SYSCTL_CHILDREN(node);
+
+	SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "irq_holdoff",
+	    CTLTYPE_U32 | CTLFLAG_RW, sc, 0, dpaa2_io_collect_irq_holdoff, "IU",
+	    "SWP holdoff time in ms");
+}
+
 static int
 dpaa2_io_attach(device_t dev)
 {
@@ -223,6 +272,7 @@ dpaa2_io_attach(device_t dev)
 	sc->swp = NULL;
 	sc->intr = NULL;
 	sc->irq_resource = NULL;
+	sc->irq_holdoff = 120;
 
 	/* Allocate resources. */
 	error = bus_alloc_resources(sc->dev, dpaa2_io_spec, sc->res);
@@ -308,8 +358,10 @@ dpaa2_io_attach(device_t dev)
 	sc->swp_desc.swp_cycles_ratio = 256000 /
 	    (sc->swp_desc.swp_clk / 1000000);
 
+	dpaa2_io_setup_sysctls(sc);
+
 	/* Initialize QBMan software portal. */
-	error = dpaa2_swp_init_portal(&sc->swp, &sc->swp_desc, DPAA2_SWP_DEF);
+	error = dpaa2_swp_init_portal(sc, DPAA2_SWP_DEF);
 	if (error) {
 		device_printf(dev, "%s: failed to initialize dpaa2_swp: "
 		    "error=%d\n", __func__, error);

sys/dev/dpaa2/dpaa2_io.h

@@ -100,6 +100,9 @@ struct dpaa2_io_softc {
 
 	int			 cpu;
 	cpuset_t		 cpu_mask;
+
+	/* sysctl(9) */
+	uint32_t		 irq_holdoff; /* ms */
 };
 
 extern struct resource_spec dpaa2_io_spec[];

sys/dev/dpaa2/dpaa2_swp.c

@@ -179,15 +179,15 @@ static int dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *,
 static int dpaa2_swp_cyc_diff(uint8_t, uint8_t, uint8_t);
 
 int
-dpaa2_swp_init_portal(struct dpaa2_swp **swp, struct dpaa2_swp_desc *desc,
-    uint16_t flags)
+dpaa2_swp_init_portal(struct dpaa2_io_softc *sc, uint16_t flags)
 {
 	struct dpaa2_swp *p;
+	struct dpaa2_swp_desc *desc;
 	uint32_t reg, mask_size, eqcr_pi; /* EQCR producer index */
 
-	if (!swp || !desc)
+	if (!sc)
 		return (DPAA2_SWP_STAT_EINVAL);
-
+	desc = &sc->swp_desc;
 	p = malloc(sizeof(struct dpaa2_swp), M_DPAA2_SWP,
 	    flags & DPAA2_SWP_NOWAIT_ALLOC
 	    ? (M_NOWAIT | M_ZERO)
@@ -297,11 +297,13 @@ dpaa2_swp_init_portal(struct dpaa2_swp **swp, struct dpaa2_swp_desc *desc,
 	    & p->eqcr.pi_ci_mask;
 	p->eqcr.available = p->eqcr.pi_ring_size;
 
-	/* TODO: sysctl(9) for the IRQ timeout? */
-	/* Initialize the portal with an IRQ threshold and timeout of 120us. */
-	dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, 120);
+	/*
+	 * Initialize the portal with an IRQ threshold and timeout from the
+	 * device context.
+	 */
+	dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, sc->irq_holdoff);
 
-	*swp = p;
+	sc->swp = p;
 
 	return (0);
 }

sys/dev/dpaa2/dpaa2_swp.h

@@ -232,6 +232,11 @@ enum dpaa2_fd_format {
 	DPAA2_FD_SG
 };
 
+/**
+ * Forward declaration to avoid circular include.
+ */
+struct dpaa2_io_softc;
+
 /**
  * @brief Enqueue command descriptor.
  */
@@ -441,8 +446,7 @@ struct dpaa2_swp {
 };
 
 /* Management routines. */
-int dpaa2_swp_init_portal(struct dpaa2_swp **swp, struct dpaa2_swp_desc *desc,
-    uint16_t flags);
+int dpaa2_swp_init_portal(struct dpaa2_io_softc *softc, uint16_t flags);
 void dpaa2_swp_free_portal(struct dpaa2_swp *swp);
 uint32_t dpaa2_swp_set_cfg(uint8_t max_fill, uint8_t wn, uint8_t est,
     uint8_t rpm, uint8_t dcm, uint8_t epm, int sd, int sp, int se, int dp,