tcp: refactor tcp_autosndbuf() and associated sysctls

Rename tcp_sndbuf_autoscale() to tcp_autosndbuf().  This matches
complementary tcp_autorcvbuf() that was factored out as a separate
function a few months before tcp_sndbuf_autoscale() was.  Move the large
comment to the header of the function.  Use unsigned types for all sizing
sysctls and variables and use bool for the switching sysctl.  Add a
comment above net.inet.tcp.sendbuf_auto_lowat (default to off) providing a
reference to its origins.  Add DTrace probe tcp::send-autoresize,
complementary to the existing tcp::receive-autoresize.

Rack has its own function, just enable the DTrace probe there.

Reviewed by:		tuexen
Differential Revision:	https://reviews.freebsd.org/D60732

sys/netinet/in_kdtrace.c

@@ -543,6 +543,10 @@ SDT_PROBE_DEFINE2_XLATE(tcp, , , receive__autoresize,
     "struct tcpcb *", "tcpsinfo_t *" ,
     "u_int", "u_int");
 
+SDT_PROBE_DEFINE2_XLATE(tcp, , , send__autoresize,
+    "struct tcpcb *", "tcpsinfo_t *" ,
+    "u_int", "u_int");
+
 SDT_PROBE_DEFINE5_XLATE(udp, , , receive,
     "void *", "pktinfo_t *",
     "struct inpcb *", "csinfo_t *",

sys/netinet/in_kdtrace.h

@@ -442,6 +442,7 @@ SDT_PROBE_DECLARE(tcp, , , debug__output);
 SDT_PROBE_DECLARE(tcp, , , debug__user);
 SDT_PROBE_DECLARE(tcp, , , debug__drop);
 SDT_PROBE_DECLARE(tcp, , , receive__autoresize);
+SDT_PROBE_DECLARE(tcp, , , send__autoresize);
 
 SDT_PROBE_DECLARE(udp, , , receive);
 SDT_PROBE_DECLARE(udp, , , send);

sys/netinet/tcp_output.c

@@ -107,31 +107,31 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_VNET | CTLFLAG_RW,
 	&VNET_NAME(tcp_do_tso), 0,
 	"Enable TCP Segmentation Offload");
 
-VNET_DEFINE(int, tcp_sendspace) = 1024*32;
+VNET_DEFINE(u_int, tcp_sendspace) = 1024*32;
 #define	V_tcp_sendspace	VNET(tcp_sendspace)
-SYSCTL_INT(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, CTLFLAG_VNET | CTLFLAG_RW,
-	&VNET_NAME(tcp_sendspace), 0, "Initial send socket buffer size");
+SYSCTL_UINT(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace,
+    CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_sendspace), 0,
+    "Initial send socket buffer size");
 
-VNET_DEFINE(int, tcp_do_autosndbuf) = 1;
-SYSCTL_INT(_net_inet_tcp, OID_AUTO, sendbuf_auto, CTLFLAG_VNET | CTLFLAG_RW,
-	&VNET_NAME(tcp_do_autosndbuf), 0,
-	"Enable automatic send buffer sizing");
+VNET_DEFINE(bool, tcp_do_autosndbuf) = true;
+SYSCTL_BOOL(_net_inet_tcp, OID_AUTO, sendbuf_auto, CTLFLAG_VNET | CTLFLAG_RW,
+    &VNET_NAME(tcp_do_autosndbuf), 0, "Enable automatic send buffer sizing");
 
-VNET_DEFINE(int, tcp_autosndbuf_inc) = 8*1024;
-SYSCTL_INT(_net_inet_tcp, OID_AUTO, sendbuf_inc, CTLFLAG_VNET | CTLFLAG_RW,
-	&VNET_NAME(tcp_autosndbuf_inc), 0,
-	"Incrementor step size of automatic send buffer");
+VNET_DEFINE(u_int, tcp_autosndbuf_inc) = 8*1024;
+SYSCTL_UINT(_net_inet_tcp, OID_AUTO, sendbuf_inc, CTLFLAG_VNET | CTLFLAG_RW,
+    &VNET_NAME(tcp_autosndbuf_inc), 0,
+    "Incrementor step size of automatic send buffer");
 
-VNET_DEFINE(int, tcp_autosndbuf_max) = 8*1024*1024;
-SYSCTL_INT(_net_inet_tcp, OID_AUTO, sendbuf_max, CTLFLAG_VNET | CTLFLAG_RW,
-	&VNET_NAME(tcp_autosndbuf_max), 0,
-	"Max size of automatic send buffer");
+VNET_DEFINE(u_int, tcp_autosndbuf_max) = 8*1024*1024;
+SYSCTL_UINT(_net_inet_tcp, OID_AUTO, sendbuf_max, CTLFLAG_VNET | CTLFLAG_RW,
+    &VNET_NAME(tcp_autosndbuf_max), 0, "Max size of automatic send buffer");
 
-VNET_DEFINE(int, tcp_sendbuf_auto_lowat) = 0;
+/* See D11016, used by LimeLight. */
+VNET_DEFINE_STATIC(bool, tcp_sendbuf_auto_lowat) = false;
 #define	V_tcp_sendbuf_auto_lowat	VNET(tcp_sendbuf_auto_lowat)
-SYSCTL_INT(_net_inet_tcp, OID_AUTO, sendbuf_auto_lowat, CTLFLAG_VNET | CTLFLAG_RW,
-	&VNET_NAME(tcp_sendbuf_auto_lowat), 0,
-	"Modify threshold for auto send buffer growth to account for SO_SNDLOWAT");
+SYSCTL_BOOL(_net_inet_tcp, OID_AUTO, sendbuf_auto_lowat,
+    CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_sendbuf_auto_lowat), 0,
+    "Modify threshold for auto send buffer growth to account for SO_SNDLOWAT");
 
 /*
  * Make sure that either retransmit or persist timer is set for SYN, FIN and
@@ -508,7 +508,7 @@ after_sack_rexmit:
 	/* len will be >= 0 after this point. */
 	KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
 
-	tcp_sndbuf_autoscale(tp, so, sendwin);
+	tcp_autosndbuf(tp, sendwin);
 
 	/*
 	 * Decide if we can use TCP Segmentation Offloading (if supported by
@@ -2119,61 +2119,59 @@ nospace:
 	return (NULL);
 }
 
+/*
+ * Automatic sizing of send socket buffer.  Often the send buffer size is not
+ * optimally adjusted to the actual network conditions at hand (delay bandwidth
+ * product).  Setting the buffer size too small limits throughput on links with
+ * high bandwidth and high delay (eg. trans-continental/oceanic links).
+ * Setting the buffer size too big consumes too much real kernel memory,
+ * especially with many connections on busy servers.
+ *
+ * The criteria to step up the send buffer one notch are:
+ *  1. receive window of remote host is larger than send buffer
+ *     (with a fudge factor of 5/4th);
+ *  2. send buffer is filled to 7/8th with data (so we actually
+ *     have data to make use of it);
+ *  3. send buffer fill has not hit maximal automatic size;
+ *  4. our send window (slow start and cogestion controlled) is
+ *     larger than sent but unacknowledged data in send buffer.
+ *
+ * The remote host receive window scaling factor may limit the growing of the
+ * send buffer before it reaches its allowed maximum.
+ *
+ * It scales directly with slow start or congestion window and does at most one
+ * step per received ACK.  This fast scaling has the drawback of growing the
+ * send buffer beyond what is strictly necessary to make full use of a given
+ * delay*bandwidth product.  However testing has shown this not to be much of
+ * an problem.  At worst we are trading wasting of available bandwidth (the
+ * non-use of it) for wasting some socket buffer memory.
+ *
+ * TODO: Shrink send buffer during idle periods together with congestion window.
+ */
 void
-tcp_sndbuf_autoscale(struct tcpcb *tp, struct socket *so, uint32_t sendwin)
+tcp_autosndbuf(struct tcpcb *tp, u_int sendwin)
 {
+	struct socket *so = tptosocket(tp);
+	u_int lowat, newsize;
 
-	/*
-	 * Automatic sizing of send socket buffer.  Often the send buffer
-	 * size is not optimally adjusted to the actual network conditions
-	 * at hand (delay bandwidth product).  Setting the buffer size too
-	 * small limits throughput on links with high bandwidth and high
-	 * delay (eg. trans-continental/oceanic links).  Setting the
-	 * buffer size too big consumes too much real kernel memory,
-	 * especially with many connections on busy servers.
-	 *
-	 * The criteria to step up the send buffer one notch are:
-	 *  1. receive window of remote host is larger than send buffer
-	 *     (with a fudge factor of 5/4th);
-	 *  2. send buffer is filled to 7/8th with data (so we actually
-	 *     have data to make use of it);
-	 *  3. send buffer fill has not hit maximal automatic size;
-	 *  4. our send window (slow start and cogestion controlled) is
-	 *     larger than sent but unacknowledged data in send buffer.
-	 *
-	 * The remote host receive window scaling factor may limit the
-	 * growing of the send buffer before it reaches its allowed
-	 * maximum.
-	 *
-	 * It scales directly with slow start or congestion window
-	 * and does at most one step per received ACK.  This fast
-	 * scaling has the drawback of growing the send buffer beyond
-	 * what is strictly necessary to make full use of a given
-	 * delay*bandwidth product.  However testing has shown this not
-	 * to be much of an problem.  At worst we are trading wasting
-	 * of available bandwidth (the non-use of it) for wasting some
-	 * socket buffer memory.
-	 *
-	 * TODO: Shrink send buffer during idle periods together
-	 * with congestion window.  Requires another timer.  Has to
-	 * wait for upcoming tcp timer rewrite.
-	 *
-	 * XXXGL: should there be used sbused() or sbavail()?
-	 */
-	if (V_tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE) {
-		int lowat;
+	SOCK_RECVBUF_LOCK_ASSERT(so);
 
-		lowat = V_tcp_sendbuf_auto_lowat ? so->so_snd.sb_lowat : 0;
-		if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat - lowat &&
-		    sbused(&so->so_snd) >=
-		    (so->so_snd.sb_hiwat / 8 * 7) - lowat &&
-		    sbused(&so->so_snd) < V_tcp_autosndbuf_max &&
-		    sendwin >= (sbused(&so->so_snd) -
-		    (tp->snd_nxt - tp->snd_una))) {
-			if (!sbreserve_locked(so, SO_SND,
-			    min(so->so_snd.sb_hiwat + V_tcp_autosndbuf_inc,
-			     V_tcp_autosndbuf_max), curthread))
-				so->so_snd.sb_flags &= ~SB_AUTOSIZE;
+	if (!V_tcp_do_autosndbuf || (so->so_snd.sb_flags & SB_AUTOSIZE) == 0)
+		return;
+
+
+	lowat = V_tcp_sendbuf_auto_lowat ? so->so_snd.sb_lowat : 0;
+	if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat - lowat &&
+	    sbused(&so->so_snd) >=
+	    (so->so_snd.sb_hiwat / 8 * 7) - lowat &&
+	    sbused(&so->so_snd) < V_tcp_autosndbuf_max &&
+	    sendwin >= (sbused(&so->so_snd) - (tp->snd_nxt - tp->snd_una))) {
+		newsize = min(so->so_snd.sb_hiwat + V_tcp_autosndbuf_inc,
+		    V_tcp_autosndbuf_max);
+		if (sbreserve_locked(so, SO_SND, newsize, curthread)) {
+			TCP_PROBE2(send__autoresize, tp, newsize);
+		} else {
+			so->so_snd.sb_flags &= ~SB_AUTOSIZE;
 		}
 	}
 }

sys/netinet/tcp_stacks/bbr.c

@@ -12510,7 +12510,7 @@ recheck_resend:
 	}
 	/* len will be >= 0 after this point. */
 	KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
-	tcp_sndbuf_autoscale(tp, so, sendwin);
+	tcp_autosndbuf(tp, sendwin);
 	/*
 	 *
 	 */

sys/netinet/tcp_stacks/rack.c

@@ -18915,8 +18915,11 @@ rack_sndbuf_autoscale(struct tcp_rack *rack)
 			scaleup += so->so_snd.sb_hiwat;
 			if (scaleup > V_tcp_autosndbuf_max)
 				scaleup = V_tcp_autosndbuf_max;
-			if (!sbreserve_locked(so, SO_SND, scaleup, curthread))
+			if (sbreserve_locked(so, SO_SND, scaleup, curthread)) {
+				TCP_PROBE2(send__autoresize, tp, scaleup);
+			} else {
 				so->so_snd.sb_flags &= ~SB_AUTOSIZE;
+			}
 		}
 	}
 }

sys/netinet/tcp_var.h

@@ -1304,12 +1304,12 @@ VNET_DECLARE(int, tcp_abc_l_var);
 VNET_DECLARE(uint32_t, tcp_ack_war_cnt);
 VNET_DECLARE(uint32_t, tcp_ack_war_time_window);
 VNET_DECLARE(u_int, tcp_autorcvbuf_max);
-VNET_DECLARE(int, tcp_autosndbuf_inc);
-VNET_DECLARE(int, tcp_autosndbuf_max);
+VNET_DECLARE(u_int, tcp_autosndbuf_inc);
+VNET_DECLARE(u_int, tcp_autosndbuf_max);
 VNET_DECLARE(int, tcp_bind_all_fibs);
 VNET_DECLARE(int, tcp_delack_enabled);
-VNET_DECLARE(int, tcp_do_autosndbuf);
 VNET_DECLARE(bool, tcp_do_autorcvbuf);
+VNET_DECLARE(bool, tcp_do_autosndbuf);
 VNET_DECLARE(int, tcp_do_ecn);
 VNET_DECLARE(int, tcp_do_lrd);
 VNET_DECLARE(int, tcp_do_prr);
@@ -1343,7 +1343,7 @@ VNET_DECLARE(int, tcp_sack_globalmaxholes);
 VNET_DECLARE(int, tcp_sack_maxholes);
 VNET_DECLARE(int, tcp_sack_tso);
 VNET_DECLARE(int, tcp_sc_rst_sock_fail);
-VNET_DECLARE(int, tcp_sendspace);
+VNET_DECLARE(u_int, tcp_sendspace);
 VNET_DECLARE(int, tcp_udp_tunneling_overhead);
 VNET_DECLARE(int, tcp_udp_tunneling_port);
 VNET_DECLARE(struct inpcbinfo, tcbinfo);
@@ -1404,6 +1404,7 @@ VNET_DECLARE(struct hhook_head *, tcp_hhh[HHOOK_TCP_LAST + 1]);
 #endif
 
 void	tcp_autorcvbuf(struct tcpcb *, u_int);
+void	tcp_autosndbuf(struct tcpcb *, u_int);
 void	tcp_account_for_send(struct tcpcb *, uint32_t, uint8_t, uint8_t, bool);
 int	 tcp_addoptions(struct tcpopt *, u_char *);
 struct tcpcb *
@@ -1553,7 +1554,6 @@ void	 tcp_sack_lost_retransmission(struct tcpcb *, struct tcphdr *);
 int	 tcp_newreno(struct tcpcb *, struct tcphdr *);
 int	 tcp_compute_pipe(struct tcpcb *);
 uint32_t tcp_compute_initwnd(uint32_t);
-void	 tcp_sndbuf_autoscale(struct tcpcb *, struct socket *, uint32_t);
 int	 tcp_stats_sample_rollthedice(struct tcpcb *tp, void *seed_bytes,
     size_t seed_len);
 int tcp_can_enable_pacing(void);